diff --git a/.gitignore b/.gitignore index e050e2c8..296400b3 100644 --- a/.gitignore +++ b/.gitignore @@ -103,6 +103,15 @@ Temporary Items .openviking *.code-workspace +# Benchmark outputs +examples/benchmark/outputs/ +examples/benchmark/datasets/full/ +examples/benchmark/*.log +RAGbenchmark/datasets/*/ +!RAGbenchmark/datasets/Benchmark_Lite/ +RAGbenchmark/Output/ +RAGbenchmark/*.log + # AI Coding CLAUDE.md *.so diff --git a/benchmark/RAG/README.md b/benchmark/RAG/README.md new file mode 100644 index 00000000..4b18aab3 --- /dev/null +++ b/benchmark/RAG/README.md @@ -0,0 +1,1052 @@ +# RAG + +## English Version + +RAG is an independent RAG (Retrieval-Augmented Generation) system evaluation framework, fully compatible with the latest version of OpenViking. + +### Project Structure + +``` +benchmark/RAG/ +├── src/ +│ ├── __init__.py +│ ├── pipeline.py # Evaluation core pipeline +│ ├── adapters/ # Dataset adapters +│ │ ├── __init__.py +│ │ ├── base.py # Base adapter class +│ │ ├── locomo_adapter.py # Locomo dataset adapter +│ │ ├── syllabusqa_adapter.py # SyllabusQA dataset adapter +│ │ ├── qasper_adapter.py # Qasper dataset adapter +│ │ ├── financebench_adapter.py # FinanceBench dataset adapter +│ │ └── ... +│ └── core/ # Core components +│ ├── __init__.py +│ ├── logger.py # Logging module +│ ├── vector_store.py # Vector store wrapper +│ ├── llm_client.py # LLM client wrapper +│ ├── metrics.py # Metrics calculation +│ ├── judge_util.py # LLM judge utility +│ └── monitor.py # Monitoring utility +├── config/ # Configuration files +│ ├── config.yaml +│ ├── locomo_config.yaml +│ ├── syllabusqa_config.yaml +│ ├── qasper_config.yaml +│ └── financebench_config.yaml +├── datasets/ # Dataset directory +│ └── Benchmark_Lite/ # Lightweight evaluation dataset +├── Output/ # Output result directory +├── run.py # Main execution script +└── README.md +``` + +### Quick Start + +#### 1. Install Dependencies + +```bash +cd OpenViking +uv pip install -e ".[benchmark]" +source .venv/bin/activate +``` + +#### 2. Prepare Datasets + +Use the lightweight datasets in `datasets/Benchmark_Lite/` for quick testing. + +#### 3. Configure LLM + +Edit LLM configuration in `config/*.yaml`. This configuration is used for both: + +- **Answer generation**: Generating answers from retrieved context +- **LLM-as-judge evaluation**: Using LLM to evaluate the quality of generated answers + +#### 4. Configure OpenViking + +If you need to use custom OpenViking configuration (for data ingestion and retrieval), create an `ov.conf` file in the benchmark/RAG directory. This will override the default OpenViking settings. + +You can refer to `examples/ov.conf.example` in the OpenViking root directory for the configuration format. + +#### 5. Run Evaluation + +```bash +cd benchmark/RAG + +# Run complete evaluation (data ingestion, answer generation, evaluation, and data deletion) +python run.py --config config/locomo_config.yaml + +# Only run data ingestion and answer generation stage +python run.py --config config/locomo_config.yaml --step gen + +# Only run evaluation stage (requires generated answers from previous step) +python run.py --config config/locomo_config.yaml --step eval + +# Only run data deletion stage +python run.py --config config/locomo_config.yaml --step del +``` + +### Supported Datasets + +| Dataset | Type | Docs | QAs | Characteristics | +| -------------- | ---------- | ---- | --- | ---------------------------------------------------------------------------------------------------------------------------- | +| **Locomo** | Multi-turn | 3 | 100 | Real user conversations, 4 question types (factual, temporal, reasoning, understanding) | +| **SyllabusQA** | Syllabus | 8 | 120 | Education domain, 6 question types (single factual, multi factual, single reasoning, multi reasoning, summarization, yes/no) | +| **Qasper** | Academic | 12 | 84 | Research domain, complex reasoning, 3 answer types (extractive, free-form, yes/no) | +| Financebench | Financial | 4 | 13 | Financial domain, SEC financial reports, 3 question types (domain-relevant, metrics-generated, novel-generated) | + +### How to Use Different Datasets + +Each dataset has its own configuration file in the `config/` directory. To use a specific dataset: + +1. **Choose a dataset configuration file**: + - `config/locomo_config.yaml` - For Locomo dataset + - `config/syllabusqa_config.yaml` - For SyllabusQA dataset + - `config/qasper_config.yaml` - For Qasper dataset + - `config/financebench_config.yaml` - For FinanceBench dataset +2. **Run evaluation with the chosen configuration**: + ```bash + # Evaluate with Locomo dataset + python run.py --config config/locomo_config.yaml + + # Evaluate with SyllabusQA dataset + python run.py --config config/syllabusqa_config.yaml + + # Evaluate with Qasper dataset + python run.py --config config/qasper_config.yaml + + # Evaluate with FinanceBench dataset + python run.py --config config/financebench_config.yaml + ``` +3. **Customize configuration (optional)**: + You can copy a dataset configuration file and modify it to suit your needs: + ```bash + cp config/locomo_config.yaml config/my_custom_config.yaml + # Edit config/my_custom_config.yaml with your preferences + python run.py --config config/my_custom_config.yaml + ``` + +### Configuration Guide + +RAG uses YAML configuration files to control the evaluation process. Each dataset has its own configuration file in the `config/` directory. + +**Key Configuration Sections:** + +1. **Basic Configuration**: + - `dataset_name`: Name of the dataset being evaluated +2. **Adapter Configuration**: + - `adapter.module`: Python module path for the dataset adapter + - `adapter.class_name`: Class name of the dataset adapter +3. **Execution Configuration**: + - `max_workers`: Number of concurrent worker threads + - `ingest_workers`: Number of worker threads for document ingestion + - `retrieval_topk`: Number of documents to retrieve + - `max_queries`: Limit the number of queries to process (null = all) + - `skip_ingestion`: Skip document ingestion (use existing index) + - `ingest_mode`: Document ingestion mode ("directory" or "per\_file") + - `retrieval_instruction`: Custom instruction for retrieval (empty by default) +4. **Path Configuration**: + - `raw_data`: Path to raw dataset file + - `doc_output_dir`: Directory for processed documents + - `vector_store`: Directory for vector index storage + - `output_dir`: Directory for evaluation results + - `log_file`: Path to log file +5. **LLM Configuration**: + - `llm.model`: LLM model name + - `llm.temperature`: Generation temperature + - `llm.base_url`: API base URL + - `llm.api_key`: API key (keep secure) + +### Evaluation Process Overview + +The evaluation process consists of 5 main stages: + +1. **Data Preparation** + - Convert raw dataset into OpenViking-friendly format + - Process documents for ingestion +2. **Data Ingestion** + - Ingest processed documents into OpenViking vector store + - Create embeddings for documents + - Store vector index for retrieval +3. **Answer Generation** + - For each question, retrieve relevant documents from vector store + - Build prompt with retrieved context and question + - Generate answer using LLM +4. **Evaluation** + - Use LLM-as-judge to evaluate generated answers against gold answers + - Calculate metrics (Recall, F1, Accuracy) +5. **Data Deletion** + - Clean up vector store and remove ingested documents + +### Evaluation Metrics + +- **Recall**: Retrieval recall rate +- **F1 Score**: Answer F1 score +- **Accuracy**: LLM judge score (0-4) +- **Latency**: Retrieval latency +- **Token Usage**: Token consumption + +### Benchmark Results + +Below are the benchmark results across different datasets for reference and comparison. + +#### Performance Metrics Comparison + +```mermaid +--- +config: + xyChart: + plotColorPalette: "#4E79A7, #F28E2B, #E15759" +--- +xychart-beta + title "Performance Metrics by Dataset" + x-axis ["Locomo", "SyllabusQA", "Qasper", "Financebench"] + y-axis "Score (0-1)" 0 --> 1 + bar "Recall" [0.72, 0.73, 0.65, 0.73] + bar "F1 Score" [0.32, 0.34, 0.44, 0.30] + line "Normalized Accuracy" [0.59, 0.74, 0.69, 0.69] +``` + +**Legend:** +- Blue Bar "Recall": Retrieval recall rate +- Green Bar "F1 Score": Answer F1 score +- Red Line "Normalized Accuracy": LLM judge score normalized to 0-1 scale + +#### Benchmark Results Table + +| Metric | Locomo | SyllabusQA | Qasper | Financebench | +| ---------------------------- | ------ | ---------- | ------ | ------------ | +| **Recall** | 0.72 | 0.73 | 0.65 | 0.73 | +| **F1 Score** | 0.32 | 0.34 | 0.44 | 0.30 | +| **Accuracy (0-4)** | 2.36 | 2.96 | 2.77 | 2.77 | +| **Normalized Accuracy** | 0.59 | 0.74 | 0.69 | 0.69 | +| **Avg Retrieval Time (s)** | 0.17 | 1.00 | 0.18 | 0.55 | +| **Avg Input Tokens** | 3364 | 2171 | 2160 | 6813 | +| **Avg Output Tokens** | 16 | 69 | 57 | 63 | +| **Total Queries** | 100 | 120 | 79 | 13 | +| **Total Insertion Time (s)** | 132 | 98 | 112 | 456 | + +**Notes:** + +- Results are based on default configuration with `retrieval_topk=5` +- FinanceBench uses `retrieval_topk=10` in this test +- Normalized Accuracy = Accuracy / 4 + +### Output Files + +Evaluation results are saved in the `Output/` directory with the following structure: + +``` +Output/ +└── {dataset_name}/ + └── experiment_{experiment_name}/ + ├── generated_answers.json # Generated answers from LLM + ├── qa_eval_detailed_results.json # Detailed evaluation results + ├── benchmark_metrics_report.json # Aggregated metrics report + ├── docs/ # Processed documents (if skip_ingestion=false) + └── benchmark.log # Log file +``` + +**Vector Store Database Location:** +The vector index (document database) is stored in the path specified by `vector_store` in the configuration file. By default, this is: + +``` +datasets/{dataset_name}/viking_store_index_dir +``` + +#### File descriptions and examples + +**1.** **`benchmark_metrics_report.json`** **- Summary Report** + +- **What it contains**: Aggregated metrics report with overall performance scores + +Example: + +```json +{ + "Insertion Efficiency (Total Dataset)": { + "Total Insertion Time (s)": 131.98, + "Total Input Tokens": 142849, + "Total Output Tokens": 52077, + "Total Embedding Tokens": 95626 + }, + "Query Efficiency (Average Per Query)": { + "Average Retrieval Time (s)": 0.17, + "Average Input Tokens": 3364.46, + "Average Output Tokens": 15.5 + }, + "Dataset": "Locomo", + "Total Queries Evaluated": 100, + "Performance Metrics": { + "Average F1 Score": 0.318, + "Average Recall": 0.724, + "Average Accuracy (Hit 0-4)": 2.36, + "Average Accuracy (normalization)": 0.59 + } +} +``` + +**Field descriptions:** + +- `Insertion Efficiency`: Document ingestion performance statistics +- `Query Efficiency`: Per-query performance averages +- `Performance Metrics`: Core evaluation scores (0-4 scale for Accuracy) + +*** + +**2.** **`generated_answers.json`** **- Generated Answers** + +- **What it contains**: All questions, retrieved contexts, and LLM-generated answers + +Example (single result): + +```json +{ + "_global_index": 0, + "sample_id": "conv-26", + "question": "Would Caroline pursue writing as a career option?", + "gold_answers": ["LIkely no; though she likes reading, she wants to be a counselor"], + "category": "3", + "evidence": ["D7:5", "D7:9"], + "retrieval": { + "latency_sec": 0.288, + "uris": ["viking://resources/...", "viking://resources/..."] + }, + "llm": { + "final_answer": "Not mentioned" + }, + "metrics": { + "Recall": 1.0 + }, + "token_usage": { + "total_input_tokens": 2643, + "llm_output_tokens": 2 + } +} +``` + +**Field descriptions:** + +- `_global_index`: Unique query identifier +- `question`: The question being asked +- `gold_answers`: Ground truth answers +- `retrieval.uris`: URIs of retrieved documents +- `llm.final_answer`: Answer generated by LLM +- `metrics.Recall`: Retrieval recall score (0-1) +- `token_usage`: Token consumption statistics + +*** + +**3.** **`qa_eval_detailed_results.json`** **- Detailed Evaluation** + +- **What it contains**: Per-question evaluation including LLM judge reasoning and scores + +Example (single result): + +```json +{ + "_global_index": 2, + "question": "When did Caroline go to the LGBTQ support group?", + "gold_answers": ["7 May 2023"], + "llm": { + "final_answer": "7 May 2023 (the day before the chat at 1:56 pm on 8 May, 2023)" + }, + "metrics": { + "Recall": 1.0, + "F1": 0.375, + "Accuracy": 4 + }, + "llm_evaluation": { + "prompt_used": "Locomo_0or4", + "reasoning": "The generated answer explicitly includes the exact date 7 May 2023 that matches the gold answer...", + "normalized_score": 4 + } +} +``` + +**Field descriptions:** + +- `metrics.F1`: Answer F1 score (0-1) +- `metrics.Accuracy`: LLM judge score (0-4, 4 = perfect) +- `llm_evaluation.reasoning`: LLM judge's reasoning for the score +- `llm_evaluation.normalized_score`: Final normalized score + +*** + +**4.** **`benchmark.log`** **- Execution Log** + +- **What it contains**: Detailed execution log with timestamps, warnings, and errors +- **How to view**: Open directly in any text editor + +*** + +**5.** **`docs/`** **- Processed Documents** + +- **What it contains**: Processed documents in Markdown format (if `skip_ingestion=false`) +- **How to view**: Open `.md` files directly in any Markdown viewer or text editor + +### Advanced Configuration + +#### Retrieval Instruction Configuration + +You can configure a custom retrieval instruction in the `config.yaml` file to guide the retrieval process. This instruction is prepended to each query during retrieval. + +**Configuration Example:** + +```yaml +# ===========Execution Configuration============ +# Instruction for retrieval, empty by default +# Recommended format: "Target_modality: xxx.\nInstruction:xxx.\nQuery:" +retrieval_instruction: "Target_modality: text.\nInstruction:Locate the part of the conversation where the speakers discuss.\nQuery:" +``` + +**Recommended Format:** + +- `Target_modality: xxx.` - Specify the target modality (e.g., text, image, audio) +- `Instruction: xxx.` - Provide specific instructions for retrieval +- `Query:` - Mark the start of the actual query + +When `retrieval_instruction` is empty, the system will use the raw question for retrieval. + +#### Customizing Prompts + +RAG uses dataset-specific and question-type-specific prompts to guide LLM answer generation. You can customize these prompts in the adapter files under `src/adapters/` to improve evaluation results. + +##### Locomo Dataset Prompts (src/adapters/locomo\_adapter.py) + +Locomo has 4 question categories, each with specific instructions: + +- **Category 1 (Factual Extraction)**: + ``` + Extract the exact factual answer from the conversation. + - Use the exact words from the context when possible + - If multiple items, separate with commas + ``` +- **Category 2 (Time-related)**: + ``` + Answer the time-related question. + - Pay close attention to DATE labels in the conversation + - Calculate relative time (e.g., "10 years ago") when needed + - Use the exact dates from the context + ``` +- **Category 3 (Reasoning)**: + ``` + Reason and infer based on the conversation. + - Use ONLY the facts in the context + - State your conclusion clearly (e.g., "Likely yes", "Probably no") + - Do NOT explain your reasoning or provide any basis/justification + - Only output your final conclusion, nothing else + - Do NOT invent information + ``` +- **Category 4 (Understanding/Significance)**: + ``` + Understand the meaning and significance. + - Focus on what the speakers mean, not just what they say + - Identify symbolism or implied meaning + - Use wording from the context when possible + ``` + +##### SyllabusQA Dataset Prompts (src/adapters/syllabusqa\_adapter.py) + +SyllabusQA has 6 question types: + +- **single factual**: Extract single factual answer +- **multi factual**: Extract multiple factual answers +- **single reasoning**: Simple logical reasoning +- **multi reasoning**: Complex reasoning +- **summarization**: Summarize relevant information +- **yes/no**: Yes/No questions + +##### Qasper Dataset Prompts (src/adapters/qasper\_adapter.py) + +Qasper has 3 answer types: + +- **extractive**: Extract exact answer from paper +- **free\_form**: Free-form answer in own words +- **yes\_no**: Yes/No questions + +##### FinanceBench Dataset Prompts (src/adapters/financebench\_adapter.py) + +FinanceBench has 3 question types: + +- **domain-relevant**: Financial domain questions +- **metrics-generated**: Calculate financial metrics +- **novel-generated**: Novel financial questions + +##### How to Customize Prompts + +1. Open the adapter file for your dataset (e.g., `src/adapters/locomo_adapter.py`) +2. Locate the `CATEGORY_INSTRUCTIONS` dictionary +3. Modify the prompt text for the question type(s) you want to improve +4. Re-run the evaluation with the modified prompts + +### Adding New Datasets + +1. Create a new adapter class in `src/adapters/`, inheriting from `BaseAdapter` +2. Create corresponding configuration file in `config/` +3. Implement necessary methods: + - `data_prepare()`: Data preprocessing + - `load_and_transform()`: Load and transform data + - `build_prompt()`: Build prompt + - `post_process_answer()`: Post-process answer + +### Integration with OpenViking + +This project integrates with OpenViking through: + +- Using `openviking` client for data ingestion and retrieval +- Configuring OpenViking connection via `ov.conf` +- Supporting dynamic loading of OpenViking's latest features + +### Frequently Asked Questions (FAQ) + +**Q: How do I skip the data ingestion stage if I already have a vector index?** +A: Set `skip_ingestion: true` in the configuration file. This will use the existing vector index. + +**Q: Can I run only the evaluation stage without re-ingesting documents?** +A: Yes! First run `--step gen` to generate answers, then run `--step eval` to evaluate the generated answers. + +**Q: What should I do if I get an API key error?** +A: Make sure you have set a valid API key in the `llm.api_key` field of your configuration file. Keep your API key secure and do not commit it to version control. + +**Q: How can I limit the number of queries processed for testing?** +A: Set `max_queries` in the configuration file to the number of queries you want to process (e.g., `max_queries: 10`). + +**Q: What's the difference between "directory" and "per\_file" ingest modes?** +A: + +- "directory": Treats the entire directory as one document +- "per\_file": Treats each file as a separate document + +**Q: How do I customize the retrieval instruction?** +A: Set `retrieval_instruction` in the configuration file. The recommended format is: +`"Target_modality: xxx.\nInstruction:xxx.\nQuery:"` + +**Q: Where can I find the evaluation results?** +A: Results are saved in the directory specified by `output_dir` in the configuration file. By default, this is `Output/{dataset_name}/experiment_{experiment_name}/`. + +### License + +Same license as OpenViking. + +*** + +## 中文版本 + +RAG 是一个独立的 RAG (Retrieval-Augmented Generation) 系统评测框架，完全兼容最新版本的 OpenViking。 + +### 项目结构 + +``` +benchmark/RAG/ +├── src/ +│ ├── __init__.py +│ ├── pipeline.py # 评测核心管道 +│ ├── adapters/ # 数据集适配器 +│ │ ├── __init__.py +│ │ ├── base.py # 基础适配器类 +│ │ ├── locomo_adapter.py # Locomo 数据集适配器 +│ │ ├── syllabusqa_adapter.py # SyllabusQA 数据集适配器 +│ │ ├── qasper_adapter.py # Qasper 数据集适配器 +│ │ ├── financebench_adapter.py # FinanceBench 数据集适配器 +│ │ └── ... +│ └── core/ # 核心组件 +│ ├── __init__.py +│ ├── logger.py # 日志模块 +│ ├── vector_store.py # 向量存储封装 +│ ├── llm_client.py # LLM 客户端封装 +│ ├── metrics.py # 指标计算 +│ ├── judge_util.py # LLM 裁判工具 +│ └── monitor.py # 监控工具 +├── config/ # 配置文件 +│ ├── config.yaml +│ ├── locomo_config.yaml +│ ├── syllabusqa_config.yaml +│ ├── qasper_config.yaml +│ └── financebench_config.yaml +├── datasets/ # 数据集目录 +│ └── Benchmark_Lite/ # 轻量级评测数据集 +├── Output/ # 输出结果目录 +├── run.py # 主运行脚本 +└── README.md +``` + +### 快速开始 + +#### 1. 安装依赖 + +```bash +cd OpenViking +uv pip install -e ".[benchmark]" +source .venv/bin/activate +``` + +#### 2. 准备数据集 + +使用 `datasets/Benchmark_Lite/` 中的轻量级数据集进行快速测试。 + +#### 3. 配置 LLM + +编辑 `config/*.yaml` 中的 LLM 配置。该配置用于以下两个过程： + +- **答案生成**：根据检索到的上下文生成答案 +- **LLM 裁判评测**：使用 LLM 评估生成答案的质量 + +#### 4. 配置 OpenViking + +如果需要使用自定义的 OpenViking 配置（用于数据入库和检索），请在 benchmark/RAG 目录下创建 `ov.conf` 文件。这将覆盖默认的 OpenViking 设置。 + +你可以参考 OpenViking 根目录下的 `examples/ov.conf.example` 了解配置格式。 + +#### 5. 运行评测 + +```bash +cd benchmark/RAG + +# 运行完整评测流程（数据导入、答案生成、评测和数据删除） +python run.py --config config/locomo_config.yaml + +# 仅运行数据导入和答案生成阶段 +python run.py --config config/locomo_config.yaml --step gen + +# 仅运行评测阶段（需要前面步骤生成的答案） +python run.py --config config/locomo_config.yaml --step eval + +# 仅运行数据删除阶段 +python run.py --config config/locomo_config.yaml --step del +``` + +### 支持的数据集 + +| 数据集 | 类型 | 文档 | QA数 | 特点 | +| -------------- | ---- | -- | --- | ------------------------------------- | +| **Locomo** | 多轮对话 | 3 | 100 | 真实用户对话，4 种问题类型（事实、时间、推理、理解） | +| **SyllabusQA** | 教学大纲 | 8 | 120 | 教育领域，6 种问题类型（单事实、多事实、单推理、多推理、总结、是/否） | +| **Qasper** | 学术论文 | 12 | 84 | 科研领域，复杂推理，3 种答案类型（提取式、自由式、是/否） | +| Financebench | 财务领域 | 4 | 13 | 财务领域，SEC 财务报告，3 种问题类型（领域相关、指标生成、新颖生成） | + +### 如何使用不同数据集 + +每个数据集在 `config/` 目录下都有自己的配置文件。要使用特定数据集： + +1. **选择数据集配置文件**： + - `config/locomo_config.yaml` - 用于 Locomo 数据集 + - `config/syllabusqa_config.yaml` - 用于 SyllabusQA 数据集 + - `config/qasper_config.yaml` - 用于 Qasper 数据集 + - `config/financebench_config.yaml` - 用于 FinanceBench 数据集 +2. **使用所选配置运行评测**： + ```bash + # 使用 Locomo 数据集评测 + python run.py --config config/locomo_config.yaml + + # 使用 SyllabusQA 数据集评测 + python run.py --config config/syllabusqa_config.yaml + + # 使用 Qasper 数据集评测 + python run.py --config config/qasper_config.yaml + + # 使用 FinanceBench 数据集评测 + python run.py --config config/financebench_config.yaml + ``` +3. **自定义配置（可选）**： + 你可以复制数据集配置文件并根据需要修改： + ```bash + cp config/locomo_config.yaml config/my_custom_config.yaml + # 编辑 config/my_custom_config.yaml 为你的偏好设置 + python run.py --config config/my_custom_config.yaml + ``` + +### 配置指南 + +RAG 使用 YAML 配置文件来控制评测过程。每个数据集在 `config/` 目录下都有自己的配置文件。 + +**关键配置部分：** + +1. **基本配置**： + - `dataset_name`：正在评测的数据集名称 +2. **适配器配置**： + - `adapter.module`：数据集适配器的 Python 模块路径 + - `adapter.class_name`：数据集适配器的类名 +3. **执行配置**： + - `max_workers`：并发工作线程数 + - `ingest_workers`：文档入库的工作线程数 + - `retrieval_topk`：要检索的文档数量 + - `max_queries`：限制要处理的查询数量（null = 全部） + - `skip_ingestion`：跳过文档入库（使用现有索引） + - `ingest_mode`：文档入库模式（"directory" 或 "per\_file"） + - `retrieval_instruction`：自定义检索指令（默认为空） +4. **路径配置**： + - `raw_data`：原始数据集文件路径 + - `doc_output_dir`：处理后文档的目录 + - `vector_store`：向量索引存储目录 + - `output_dir`：评测结果目录 + - `log_file`：日志文件路径 +5. **LLM 配置**： + - `llm.model`：LLM 模型名称 + - `llm.temperature`：生成温度 + - `llm.base_url`：API 基础 URL + - `llm.api_key`：API 密钥（请保密） + +### 评测流程概述 + +评测流程包含 5 个主要阶段： + +1. **数据准备** + - 将原始数据集转换为 OpenViking 友好的格式 + - 处理文档以便入库 +2. **数据入库** + - 将处理后的文档入库到 OpenViking 向量存储 + - 为文档创建嵌入向量 + - 存储向量索引以便检索 +3. **答案生成** + - 对每个问题，从向量存储中检索相关文档 + - 使用检索到的上下文和问题构建提示词 + - 使用 LLM 生成答案 +4. **评测** + - 使用 LLM 裁判将生成的答案与标准答案进行对比评估 + - 计算指标（Recall、F1、Accuracy） +5. **数据删除** + - 清理向量存储并删除已入库的文档 + +### 评测指标 + +- **Recall**: 检索召回率 +- **F1 Score**: 答案 F1 值 +- **Accuracy**: LLM 裁判评分 (0-4) +- **Latency**: 检索延迟 +- **Token Usage**: Token 使用量 + +### 评测结果 + +以下是不同数据集的评测结果，供参考和对比。 + +#### 性能指标对比 + +```mermaid +xychart-beta + title "各数据集性能指标" + x-axis ["Locomo", "SyllabusQA", "Qasper", "Financebench"] + y-axis "分数 (0-1)" 0 --> 1 + bar "召回率" [0.72, 0.73, 0.65, 0.73] + bar "F1 分数" [0.32, 0.34, 0.44, 0.30] + line "归一化准确率" [0.59, 0.74, 0.69, 0.69] +``` + +**图例：** +- 蓝色柱状图 "召回率"：检索召回率 +- 绿色柱状图 "F1 分数"：答案 F1 值 +- 红色折线图 "归一化准确率"：LLM 裁判评分归一化到 0-1 范围 + +#### 评测结果表 + +| 指标 | Locomo | SyllabusQA | Qasper | Financebench | +| -------------------------------- | ------ | ---------- | ------ | ------------ | +| **召回率 (Recall)** | 0.72 | 0.73 | 0.65 | 0.73 | +| **F1 分数 (F1 Score)** | 0.32 | 0.34 | 0.44 | 0.30 | +| **准确率 (Accuracy 0-4)** | 2.36 | 2.96 | 2.77 | 2.77 | +| **归一化准确率 (Normalized Accuracy)** | 0.59 | 0.74 | 0.69 | 0.69 | +| **平均检索时间 (秒)** | 0.17 | 1.00 | 0.18 | 0.55 | +| **平均输入 Token 数** | 3364 | 2171 | 2160 | 6813 | +| **平均输出 Token 数** | 16 | 69 | 57 | 63 | +| **总查询数** | 100 | 120 | 79 | 13 | +| **总入库时间 (秒)** | 132 | 98 | 112 | 456 | + +**说明：** + +- 结果基于默认配置，`retrieval_topk=5`，FinanceBench 在此测试中使用 `retrieval_topk=10` +- 归一化准确率 = 准确率 / 4 + +### 输出文件 + +评测结果保存在 `Output/` 目录中，结构如下： + +``` +Output/ +└── {dataset_name}/ + └── experiment_{experiment_name}/ + ├── generated_answers.json # LLM 生成的答案 + ├── qa_eval_detailed_results.json # 详细的评测结果 + ├── benchmark_metrics_report.json # 汇总的指标报告 + └── benchmark.log # 日志文件 +``` + +**向量存储数据库位置：** +向量索引（文档数据库）存储在配置文件中 `vector_store` 指定的路径。默认情况下为： + +``` +datasets/{dataset_name}/viking_store_index_dir +``` + +#### 文件说明及示例 + +**1.** **`benchmark_metrics_report.json`** **- 汇总报告** + +- **包含内容**: 汇总的指标报告，包含整体性能分数 + +示例： + +```json +{ + "Insertion Efficiency (Total Dataset)": { + "Total Insertion Time (s)": 131.98, + "Total Input Tokens": 142849, + "Total Output Tokens": 52077, + "Total Embedding Tokens": 95626 + }, + "Query Efficiency (Average Per Query)": { + "Average Retrieval Time (s)": 0.17, + "Average Input Tokens": 3364.46, + "Average Output Tokens": 15.5 + }, + "Dataset": "Locomo", + "Total Queries Evaluated": 100, + "Performance Metrics": { + "Average F1 Score": 0.318, + "Average Recall": 0.724, + "Average Accuracy (Hit 0-4)": 2.36, + "Average Accuracy (normalization)": 0.59 + } +} +``` + +**字段说明：** + +- `Insertion Efficiency`: 文档入库性能统计 +- `Query Efficiency`: 每个查询的平均性能 +- `Performance Metrics`: 核心评测分数（Accuracy 为 0-4 分） + +*** + +**2.** **`generated_answers.json`** **- 生成的答案** + +- **包含内容**: 所有问题、检索到的上下文和 LLM 生成的答案 + +示例（单个结果）： + +```json +{ + "_global_index": 0, + "sample_id": "conv-26", + "question": "Would Caroline pursue writing as a career option?", + "gold_answers": ["LIkely no; though she likes reading, she wants to be a counselor"], + "category": "3", + "evidence": ["D7:5", "D7:9"], + "retrieval": { + "latency_sec": 0.288, + "uris": ["viking://resources/...", "viking://resources/..."] + }, + "llm": { + "final_answer": "Not mentioned" + }, + "metrics": { + "Recall": 1.0 + }, + "token_usage": { + "total_input_tokens": 2643, + "llm_output_tokens": 2 + } +} +``` + +**字段说明：** + +- `_global_index`: 唯一的查询标识符 +- `question`: 被询问的问题 +- `gold_answers`: 标准答案 +- `retrieval.uris`: 检索到的文档的 URI +- `llm.final_answer`: LLM 生成的答案 +- `metrics.Recall`: 检索召回分数 (0-1) +- `token_usage`: Token 消耗统计 + +*** + +**3.** **`qa_eval_detailed_results.json`** **- 详细评测结果** + +- **包含内容**: 每个问题的评测，包括 LLM 裁判推理和分数 + +示例（单个结果）： + +```json +{ + "_global_index": 2, + "question": "When did Caroline go to the LGBTQ support group?", + "gold_answers": ["7 May 2023"], + "llm": { + "final_answer": "7 May 2023 (the day before the chat at 1:56 pm on 8 May, 2023)" + }, + "metrics": { + "Recall": 1.0, + "F1": 0.375, + "Accuracy": 4 + }, + "llm_evaluation": { + "prompt_used": "Locomo_0or4", + "reasoning": "The generated answer explicitly includes the exact date 7 May 2023 that matches the gold answer...", + "normalized_score": 4 + } +} +``` + +**字段说明：** + +- `metrics.F1`: 答案 F1 分数 (0-1) +- `metrics.Accuracy`: LLM 裁判分数 (0-4, 4 = 满分) +- `llm_evaluation.reasoning`: LLM 裁判给出分数的推理 +- `llm_evaluation.normalized_score`: 最终归一化分数 + +*** + +**4.** **`benchmark.log`** **- 执行日志** + +- **包含内容**: 详细的执行日志，包含时间戳、警告和错误 +- **查看方法**: 直接在任意文本编辑器中打开 + +*** + +**5.** **`docs/`** **- 处理后的文档** + +- **包含内容**: Markdown 格式的处理后文档（如果 `skip_ingestion=false`） +- **查看方法**: 直接在任意 Markdown 查看器或文本编辑器中打开 `.md` 文件 + +### 高级配置 + +#### 检索指令配置 + +你可以在 `config.yaml` 文件中配置自定义的检索指令，以指导检索过程。该指令会在检索时被添加到每个查询的前面。 + +**配置示例：** + +```yaml +# ===========Execution Configuration============ +# 检索指令，默认为空 +# 推荐格式："Target_modality: xxx.\nInstruction:xxx.\nQuery:" +retrieval_instruction: "Target_modality: text.\nInstruction:Locate the part of the conversation where the speakers discuss.\nQuery:" +``` + +**推荐格式：** + +- `Target_modality: xxx.` - 指定目标模态（例如，文本、图像、音频） +- `Instruction: xxx.` - 提供特定的检索指令 +- `Query:` - 标记实际查询的开始 + +当 `retrieval_instruction` 为空时，系统将使用原始问题进行检索。 + +#### 自定义 Prompt + +RAG 使用数据集特定和问题类型特定的 prompt 来指导 LLM 生成答案。你可以在 `src/adapters/` 下的适配器文件中自定义这些 prompt，以改进评测结果。 + +##### Locomo 数据集 Prompt (src/adapters/locomo\_adapter.py) + +Locomo 有 4 种问题类别，每种都有特定的指令： + +- **类别 1（事实提取）**: + ``` + Extract the exact factual answer from the conversation. + - Use the exact words from the context when possible + - If multiple items, separate with commas + ``` +- **类别 2（时间相关）**: + ``` + Answer the time-related question. + - Pay close attention to DATE labels in the conversation + - Calculate relative time (e.g., "10 years ago") when needed + - Use the exact dates from the context + ``` +- **类别 3（推理）**: + ``` + Reason and infer based on the conversation. + - Use ONLY the facts in the context + - State your conclusion clearly (e.g., "Likely yes", "Probably no") + - Do NOT explain your reasoning or provide any basis/justification + - Only output your final conclusion, nothing else + - Do NOT invent information + ``` +- **类别 4（理解/意义）**: + ``` + Understand the meaning and significance. + - Focus on what the speakers mean, not just what they say + - Identify symbolism or implied meaning + - Use wording from the context when possible + ``` + +##### SyllabusQA 数据集 Prompt (src/adapters/syllabusqa\_adapter.py) + +SyllabusQA 有 6 种问题类型： + +- **single factual**: 提取单个事实答案 +- **multi factual**: 提取多个事实答案 +- **single reasoning**: 简单逻辑推理 +- **multi reasoning**: 复杂推理 +- **summarization**: 总结相关信息 +- **yes/no**: 是/否问题 + +##### Qasper 数据集 Prompt (src/adapters/qasper\_adapter.py) + +Qasper 有 3 种答案类型： + +- **extractive**: 从论文中提取精确答案 +- **free\_form**: 用自己的话自由回答 +- **yes\_no**: 是/否问题 + +##### FinanceBench 数据集 Prompt (src/adapters/financebench\_adapter.py) + +FinanceBench 有 3 种问题类型： + +- **domain-relevant**: 财务领域问题 +- **metrics-generated**: 计算财务指标 +- **novel-generated**: 新颖的财务问题 + +##### 如何自定义 Prompt + +1. 打开你的数据集的适配器文件（例如 `src/adapters/locomo_adapter.py`） +2. 找到 `CATEGORY_INSTRUCTIONS` 字典 +3. 修改你想要改进的问题类型的 prompt 文本 +4. 使用修改后的 prompt 重新运行评测 + +### 添加新数据集 + +1. 在 `src/adapters/` 中创建新的适配器类，继承自 `BaseAdapter` +2. 在 `config/` 中创建对应的配置文件 +3. 实现必要的方法： + - `data_prepare()`: 数据预处理 + - `load_and_transform()`: 加载并转换数据 + - `build_prompt()`: 构建提示词 + - `post_process_answer()`: 后处理答案 + +### 与 OpenViking 集成 + +本项目通过以下方式与 OpenViking 集成： + +- 使用 `openviking` 客户端进行数据入库和检索 +- 通过 `ov.conf` 配置 OpenViking 连接 +- 支持动态加载 OpenViking 的最新功能 + +### 常见问题解答（FAQ） + +**Q：如果我已经有向量索引，如何跳过数据入库阶段？** +A：在配置文件中设置 `skip_ingestion: true`。这将使用现有的向量索引。 + +**Q：我可以只运行评测阶段而不重新入库文档吗？** +A：可以！首先运行 `--step gen` 生成答案，然后运行 `--step eval` 来评测生成的答案。 + +**Q：如果我遇到 API 密钥错误，应该怎么办？** +A：确保你在配置文件的 `llm.api_key` 字段中设置了有效的 API 密钥。请保管好你的 API 密钥，不要将其提交到版本控制中。 + +**Q：如何限制处理的查询数量以进行测试？** +A：在配置文件中设置 `max_queries` 为你想要处理的查询数量（例如，`max_queries: 10`）。 + +**Q："directory" 和 "per\_file" 入库模式有什么区别？** +A： + +- "directory"：将整个目录视为一个文档 +- "per\_file"：将每个文件视为一个单独的文档 + +**Q：如何自定义检索指令？** +A：在配置文件中设置 `retrieval_instruction`。推荐格式为： +`"Target_modality: xxx.\nInstruction:xxx.\nQuery:"` + +**Q：我可以在哪里找到评测结果？** +A：结果保存在配置文件中 `output_dir` 指定的目录中。默认情况下，这是 `Output/{dataset_name}/experiment_{experiment_name}/`。 + +### License + +与 OpenViking 使用相同的许可证。 diff --git a/benchmark/RAG/config/config.yaml b/benchmark/RAG/config/config.yaml new file mode 100644 index 00000000..7191f66b --- /dev/null +++ b/benchmark/RAG/config/config.yaml @@ -0,0 +1,59 @@ +# RAG Benchmark Configuration File +project_name: "RAG_Benchmark" + +# ===========Basic Configuration============ +# Dataset name to identify the current dataset being evaluated +# Supported datasets: NarrativeQA, Qasper, Locomo, SyllabusQA, etc. +dataset_name: "SyllabusQA" + +# ===========Adapter Configuration============ +# Dataset adapter configuration for loading and processing dataset-specific formats +adapter: + # Adapter module path (Python import path) + module: "src.adapters.syllabusqa_adapter" + # Adapter class name + class_name: "SyllabusQAAdapter" + +# ===========Execution Configuration============ +# Maximum number of concurrent worker threads for parallel QA task processing +max_workers: 10 +# Number of concurrent worker threads for document ingestion +ingest_workers: 10 +# Number of documents to retrieve (top-k) +retrieval_topk: 5 +# Limit the maximum number of queries to process, null means process all queries +max_queries: null +# Whether to skip document ingestion phase (true means use existing vector index) +skip_ingestion: false +# Document ingestion mode +# - "directory": Unified directory ingestion mode, treats the entire directory as one document +# - "per_file": Per-file ingestion mode, each file is treated as a separate document +ingest_mode: "per_file" +# Instruction for retrieval, empty by default +retrieval_instruction: "" + +# ===========Path Configuration============ +# All paths support {dataset_name} placeholder, which will be automatically replaced with the current dataset name +paths: + # Raw data file path + raw_data: "datasets/Benchmark_Lite/{dataset_name}/test.csv" + # Output directory for processed documents + doc_output_dir: "datasets/{dataset_name}/{dataset_name}_processed_docs" + # Vector index storage directory + vector_store: "datasets/{dataset_name}/viking_store_index_dir" + # Results output directory + output_dir: "Output/{dataset_name}/experiment_dev" + # Log file path + log_file: "Output/{dataset_name}/experiment_dev/benchmark.log" + +# ===========LLM Configuration============ +# Large language model configuration for answer generation and evaluation +llm: + # Model name + model: "doubao-seed-2-0-pro-260215" + # Generation temperature (0 means deterministic output, higher values mean more randomness) + temperature: 0 + # API base URL + base_url: "https://ark.cn-beijing.volces.com/api/v3" + # API key (keep this secure, do not commit to code repositories) + api_key: "" diff --git a/benchmark/RAG/config/financebench_config.yaml b/benchmark/RAG/config/financebench_config.yaml new file mode 100644 index 00000000..7a41fad5 --- /dev/null +++ b/benchmark/RAG/config/financebench_config.yaml @@ -0,0 +1,32 @@ +project_name: "RAG_Benchmark" + +# ===========Modify Configuration================ +dataset_name: "Financebench" + +adapter: + module: "src.adapters.financebench_adapter" + class_name: "FinanceBenchAdapter" + +execution: + max_workers: 8 + ingest_workers: 8 + retrieval_topk: 10 + max_queries: null + skip_ingestion: false + ingest_mode: "directory" + retrieval_instruction: "" +# ================================== + +paths: + raw_data: "datasets/Benchmark_Lite/{dataset_name}/financebench_subset.jsonl" + doc_output_dir: "datasets/{dataset_name}/{dataset_name}_processed_docs" + vector_store: "datasets/{dataset_name}/{dataset_name}_viking_store_index" + output_dir: "Output/{dataset_name}/experiment_test_top_{retrieval_topk}" + log_file: "Output/{dataset_name}/experiment_test_top_{retrieval_topk}/benchmark.log" + + +llm: + model: "doubao-seed-2-0-pro-260215" + temperature: 0 + base_url: "https://ark.cn-beijing.volces.com/api/v3" + api_key: "" diff --git a/benchmark/RAG/config/locomo_config.yaml b/benchmark/RAG/config/locomo_config.yaml new file mode 100644 index 00000000..254282b2 --- /dev/null +++ b/benchmark/RAG/config/locomo_config.yaml @@ -0,0 +1,32 @@ +project_name: "RAG_Benchmark" + +# ===========Modify Configuration================ +dataset_name: "Locomo" + +adapter: + module: "src.adapters.locomo_adapter" + class_name: "LocomoAdapter" + +execution: + max_workers: 8 + ingest_workers: 8 + retrieval_topk: 5 + max_queries: 20 + skip_ingestion: false + ingest_mode: "directory" + retrieval_instruction: "" +# ================================== + +paths: + raw_data: "datasets/Benchmark_Lite/{dataset_name}/Locomo.json" + doc_output_dir: "datasets/{dataset_name}/{dataset_name}_processed_docs" + vector_store: "datasets/{dataset_name}/{dataset_name}_viking_store_index_dir" + output_dir: "Output/{dataset_name}/experiment_test_top_{retrieval_topk}" + log_file: "Output/{dataset_name}/experiment_test_top_{retrieval_topk}/benchmark.log" + + +llm: + model: "doubao-seed-2-0-pro-260215" + temperature: 0 + base_url: "https://ark.cn-beijing.volces.com/api/v3" + api_key: "" diff --git a/benchmark/RAG/config/qasper_config.yaml b/benchmark/RAG/config/qasper_config.yaml new file mode 100644 index 00000000..e7e90f13 --- /dev/null +++ b/benchmark/RAG/config/qasper_config.yaml @@ -0,0 +1,32 @@ +project_name: "RAG_Benchmark" + +# ===========Modify Configuration================ +dataset_name: "Qasper" + +adapter: + module: "src.adapters.qasper_adapter" + class_name: "QasperAdapter" + +execution: + max_workers: 8 + ingest_workers: 8 + retrieval_topk: 5 + max_queries: 10 + skip_ingestion: false + ingest_mode: "directory" + retrieval_instruction: "" +# ================================== + +paths: + raw_data: "datasets/Benchmark_Lite/{dataset_name}/qasper-dev-v0.3.json" + doc_output_dir: "datasets/{dataset_name}/{dataset_name}_processed_docs" + vector_store: "datasets/{dataset_name}/{dataset_name}_viking_store_index" + output_dir: "Output/{dataset_name}/experiment_test_top_{retrieval_topk}" + log_file: "Output/{dataset_name}/experiment_test_top_{retrieval_topk}/benchmark.log" + + +llm: + model: "doubao-seed-2-0-pro-260215" + temperature: 0 + base_url: "https://ark.cn-beijing.volces.com/api/v3" + api_key: "" diff --git a/benchmark/RAG/config/syllabusqa_config.yaml b/benchmark/RAG/config/syllabusqa_config.yaml new file mode 100644 index 00000000..26bd86d2 --- /dev/null +++ b/benchmark/RAG/config/syllabusqa_config.yaml @@ -0,0 +1,32 @@ +project_name: "RAG_Benchmark" + +# ===========Modify Configuration================ +dataset_name: "SyllabusQA" + +adapter: + module: "src.adapters.syllabusqa_adapter" + class_name: "SyllabusQAAdapter" + +execution: + max_workers: 8 + ingest_workers: 8 + retrieval_topk: 5 + max_queries: 20 + skip_ingestion: false + ingest_mode: "directory" + retrieval_instruction: "" +# ================================== + +paths: + raw_data: "datasets/Benchmark_Lite/{dataset_name}/test.csv" + doc_output_dir: "datasets/{dataset_name}/{dataset_name}_processed_docs" + vector_store: "datasets/{dataset_name}/{dataset_name}_viking_store_index" + output_dir: "Output/{dataset_name}/experiment_test_top_{retrieval_topk}_111" + log_file: "Output/{dataset_name}/experiment_test_top_{retrieval_topk}_111/benchmark.log" + + +llm: + model: "doubao-seed-2-0-pro-260215" + temperature: 0 + base_url: "https://ark.cn-beijing.volces.com/api/v3" + api_key: "" diff --git a/benchmark/RAG/datasets/Benchmark_Lite/Financebench/financebench_subset.jsonl b/benchmark/RAG/datasets/Benchmark_Lite/Financebench/financebench_subset.jsonl new file mode 100644 index 00000000..5b547fe3 --- /dev/null +++ b/benchmark/RAG/datasets/Benchmark_Lite/Financebench/financebench_subset.jsonl @@ -0,0 +1,13 @@ +{"financebench_id": "financebench_id_01319", "company": "AES Corporation", "doc_name": "AES_2022_10K", "question_type": "domain-relevant", "question_reasoning": "Information extraction", "domain_question_num": "dg21", "question": "What is the quantity of restructuring costs directly outlined in AES Corporation's income statements for FY2022? If restructuring costs are not explicitly outlined then state 0.", "answer": "0", "justification": null, "dataset_subset_label": "OPEN_SOURCE", "evidence": [{"evidence_text": "Consolidated Statements of Operations\nYears ended December 31, 2022, 2021, and 2020\n2022\n2021\n2020\n(in millions, except per share amounts)\nRevenue:\nRegulated\n$\n3,538 \n$\n2,868 \n$\n2,661 \nNon-Regulated\n9,079 \n8,273 \n6,999 \nTotal revenue\n12,617 \n11,141 \n9,660 \nCost of Sales:\nRegulated\n(3,162)\n(2,448)\n(2,235)\nNon-Regulated\n(6,907)\n(5,982)\n(4,732)\nTotal cost of sales\n(10,069)\n(8,430)\n(6,967)\nOperating margin\n2,548 \n2,711 \n2,693 \nGeneral and administrative expenses\n(207)\n(166)\n(165)\nInterest expense\n(1,117)\n(911)\n(1,038)\nInterest income\n389 \n298 \n268 \nLoss on extinguishment of debt\n(15)\n(78)\n(186)\nOther expense\n(68)\n(60)\n(53)\nOther income\n102 \n410 \n75 \nLoss on disposal and sale of business interests\n(9)\n(1,683)\n(95)\nGoodwill impairment expense\n(777)\n \n \nAsset impairment expense\n(763)\n(1,575)\n(864)\nForeign currency transaction gains (losses)\n(77)\n(10)\n55 \nOther non-operating expense\n(175)\n \n(202)\nINCOME (LOSS) FROM CONTINUING OPERATIONS BEFORE TAXES AND EQUITY IN EARNINGS OF AFFILIATES\n(169)\n(1,064)\n488 \nIncome tax benefit (expense)\n(265)\n133 \n(216)\nNet equity in losses of affiliates\n(71)\n(24)\n(123)\nINCOME (LOSS) FROM CONTINUING OPERATIONS\n(505)\n(955)\n149 \nGain from disposal of discontinued businesses, net of income tax expense of $0, $1, and $0, respectively\n \n4 \n3 \nNET INCOME (LOSS)\n(505)\n(951)\n152 \nLess: Net loss (income) attributable to noncontrolling interests and redeemable stock of subsidiaries\n(41)\n542 \n(106)\nNET INCOME (LOSS) ATTRIBUTABLE TO THE AES CORPORATION\n$\n(546)\n$\n(409)\n$\n46", "doc_name": "AES_2022_10K", "evidence_page_num": 131, "evidence_text_full_page": "129 \nConsolidated Statements of Operations\nYears ended December 31, 2022, 2021, and 2020\n2022\n2021\n2020\n(in millions, except per share amounts)\nRevenue:\nRegulated\n$\n3,538 \n$\n2,868 \n$\n2,661 \nNon-Regulated\n9,079 \n8,273 \n6,999 \nTotal revenue\n12,617 \n11,141 \n9,660 \nCost of Sales:\nRegulated\n(3,162)\n(2,448)\n(2,235)\nNon-Regulated\n(6,907)\n(5,982)\n(4,732)\nTotal cost of sales\n(10,069)\n(8,430)\n(6,967)\nOperating margin\n2,548 \n2,711 \n2,693 \nGeneral and administrative expenses\n(207)\n(166)\n(165)\nInterest expense\n(1,117)\n(911)\n(1,038)\nInterest income\n389 \n298 \n268 \nLoss on extinguishment of debt\n(15)\n(78)\n(186)\nOther expense\n(68)\n(60)\n(53)\nOther income\n102 \n410 \n75 \nLoss on disposal and sale of business interests\n(9)\n(1,683)\n(95)\nGoodwill impairment expense\n(777)\n \n \nAsset impairment expense\n(763)\n(1,575)\n(864)\nForeign currency transaction gains (losses)\n(77)\n(10)\n55 \nOther non-operating expense\n(175)\n \n(202)\nINCOME (LOSS) FROM CONTINUING OPERATIONS BEFORE TAXES AND EQUITY IN EARNINGS OF AFFILIATES\n(169)\n(1,064)\n488 \nIncome tax benefit (expense)\n(265)\n133 \n(216)\nNet equity in losses of affiliates\n(71)\n(24)\n(123)\nINCOME (LOSS) FROM CONTINUING OPERATIONS\n(505)\n(955)\n149 \nGain from disposal of discontinued businesses, net of income tax expense of $0, $1, and $0, respectively\n \n4 \n3 \nNET INCOME (LOSS)\n(505)\n(951)\n152 \nLess: Net loss (income) attributable to noncontrolling interests and redeemable stock of subsidiaries\n(41)\n542 \n(106)\nNET INCOME (LOSS) ATTRIBUTABLE TO THE AES CORPORATION\n$\n(546)\n$\n(409)\n$\n46 \nAMOUNTS ATTRIBUTABLE TO THE AES CORPORATION COMMON STOCKHOLDERS:\nIncome (loss) from continuing operations, net of tax\n$\n(546)\n$\n(413)\n$\n43 \nIncome from discontinued operations, net of tax\n \n4 \n3 \nNET INCOME (LOSS) ATTRIBUTABLE TO THE AES CORPORATION\n$\n(546)\n$\n(409)\n$\n46 \nBASIC EARNINGS PER SHARE:\nIncome (loss) from continuing operations attributable to The AES Corporation common stockholders, net of tax\n$\n(0.82)\n$\n(0.62)\n$\n0.06 \nIncome from discontinued operations attributable to The AES Corporation common stockholders, net of tax\n \n0.01 \n0.01 \nNET INCOME (LOSS) ATTRIBUTABLE TO THE AES CORPORATION COMMON STOCKHOLDERS\n$\n(0.82)\n$\n(0.61)\n$\n0.07 \nDILUTED EARNINGS PER SHARE:\nIncome (loss) from continuing operations attributable to The AES Corporation common stockholders, net of tax\n$\n(0.82)\n$\n(0.62)\n$\n0.06 \nIncome from discontinued operations attributable to The AES Corporation common stockholders, net of tax\n \n0.01 \n0.01 \nNET INCOME (LOSS) ATTRIBUTABLE TO THE AES CORPORATION COMMON STOCKHOLDERS\n$\n(0.82)\n$\n(0.61)\n$\n0.07 \nSee Accompanying Notes to Consolidated Financial Statements.\n"}]} +{"financebench_id": "financebench_id_00540", "company": "AES Corporation", "doc_name": "AES_2022_10K", "question_type": "domain-relevant", "question_reasoning": "Numerical reasoning OR Logical reasoning", "domain_question_num": "dg25", "question": "Roughly how many times has AES Corporation sold its inventory in FY2022? Calculate inventory turnover ratio for the FY2022; if conventional inventory management is not meaningful for the company then state that and explain why.", "answer": "AES has converted inventory 9.5 times in FY 2022.", "justification": "Cost of sales/Inventory\n10069/1055", "dataset_subset_label": "OPEN_SOURCE", "evidence": [{"evidence_text": "Consolidated Balance Sheets\nDecember 31, 2022 and 2021\n2022\n2021\n(in millions, except share and per share data)\nASSETS\nCURRENT ASSETS\nCash and cash equivalents\n$\n1,374 \n$\n943 \nRestricted cash\n536 \n304 \nShort-term investments\n730 \n232 \nAccounts receivable, net of allowance for doubtful accounts of $5 and $5, respectively\n1,799 \n1,418 \nInventory\n1,055 \n604", "doc_name": "AES_2022_10K", "evidence_page_num": 129, "evidence_text_full_page": "128 \nConsolidated Balance Sheets\nDecember 31, 2022 and 2021\n2022\n2021\n(in millions, except share and per share data)\nASSETS\nCURRENT ASSETS\nCash and cash equivalents\n$\n1,374 \n$\n943 \nRestricted cash\n536 \n304 \nShort-term investments\n730 \n232 \nAccounts receivable, net of allowance for doubtful accounts of $5 and $5, respectively\n1,799 \n1,418 \nInventory\n1,055 \n604 \nPrepaid expenses\n98 \n142 \nOther current assets, net of CECL allowance of $2 and $0, respectively\n1,533 \n897 \nCurrent held-for-sale assets\n518 \n816 \nTotal current assets\n7,643 \n5,356 \nNONCURRENT ASSETS\nProperty, Plant and Equipment:\nLand\n470 \n426 \nElectric generation, distribution assets and other\n26,599 \n25,552 \nAccumulated depreciation\n(8,651)\n(8,486)\nConstruction in progress\n4,621 \n2,414 \nProperty, plant and equipment, net\n23,039 \n19,906 \nOther Assets:\nInvestments in and advances to affiliates\n952 \n1,080 \nDebt service reserves and other deposits\n177 \n237 \nGoodwill\n362 \n1,177 \nOther intangible assets, net of accumulated amortization of $434 and $385, respectively\n1,841 \n1,450 \nDeferred income taxes\n319 \n409 \nLoan receivable, net of allowance of $26\n1,051 \n \nOther noncurrent assets, net of allowance of $51 and $23, respectively\n2,979 \n2,188 \nNoncurrent held-for-sale assets\n \n1,160 \nTotal other assets\n7,681 \n7,701 \nTOTAL ASSETS\n$\n38,363 \n$\n32,963 \nLIABILITIES AND EQUITY\nCURRENT LIABILITIES\nAccounts payable\n$\n1,730 \n$\n1,153 \nAccrued interest\n249 \n182 \nAccrued non-income taxes\n249 \n266 \nAccrued and other liabilities\n2,151 \n1,205 \nNon-recourse debt, including $416 and $302, respectively, related to variable interest entities\n1,758 \n1,367 \nCurrent held-for-sale liabilities\n354 \n559 \nTotal current liabilities\n6,491 \n4,732 \nNONCURRENT LIABILITIES\nRecourse debt\n3,894 \n3,729 \nNon-recourse debt, including $2,295 and $2,223, respectively, related to variable interest entities\n17,846 \n13,603 \nDeferred income taxes\n1,139 \n977 \nOther noncurrent liabilities\n3,168 \n3,358 \nNoncurrent held-for-sale liabilities\n \n740 \nTotal noncurrent liabilities\n26,047 \n22,407 \nCommitments and Contingencies (see Notes 12 and 13)\nRedeemable stock of subsidiaries\n1,321 \n1,257 \nEQUITY\nTHE AES CORPORATION STOCKHOLDERS EQUITY\nPreferred stock (without par value, 50,000,000 shares authorized; 1,043,050 issued and outstanding at December 31, 2022 and\nDecember 31, 2021)\n838 \n838 \nCommon stock ($0.01 par value, 1,200,000,000 shares authorized; 818,790,001 issued and 668,743,464 outstanding at December\n31, 2022 and 818,717,043 issued and 666,793,625 outstanding at December 31, 2021)\n8 \n8 \nAdditional paid-in capital\n6,688 \n7,106 \nAccumulated deficit\n(1,635)\n(1,089)\nAccumulated other comprehensive loss\n(1,640)\n(2,220)\nTreasury stock, at cost (150,046,537 and 151,923,418 shares at December 31, 2022 and December 31, 2021, respectively)\n(1,822)\n(1,845)\nTotal AES Corporation stockholders equity\n2,437 \n2,798 \nNONCONTROLLING INTERESTS\n2,067 \n1,769 \nTotal equity\n4,504 \n4,567 \nTOTAL LIABILITIES AND EQUITY\n$\n38,363 \n$\n32,963 \nSee Accompanying Notes to Consolidated Financial Statements.\n"}, {"evidence_text": "Consolidated Statements of Operations\nYears ended December 31, 2022, 2021, and 2020\n2022\n2021\n2020\n(in millions, except per share amounts)\nRevenue:\nRegulated\n$\n3,538 \n$\n2,868 \n$\n2,661 \nNon-Regulated\n9,079 \n8,273 \n6,999 \nTotal revenue\n12,617 \n11,141 \n9,660 \nCost of Sales:\nRegulated\n(3,162)\n(2,448)\n(2,235)\nNon-Regulated\n(6,907)\n(5,982)\n(4,732)\nTotal cost of sales\n(10,069)\n(8,430)\n(6,967)", "doc_name": "AES_2022_10K", "evidence_page_num": 131, "evidence_text_full_page": "129 \nConsolidated Statements of Operations\nYears ended December 31, 2022, 2021, and 2020\n2022\n2021\n2020\n(in millions, except per share amounts)\nRevenue:\nRegulated\n$\n3,538 \n$\n2,868 \n$\n2,661 \nNon-Regulated\n9,079 \n8,273 \n6,999 \nTotal revenue\n12,617 \n11,141 \n9,660 \nCost of Sales:\nRegulated\n(3,162)\n(2,448)\n(2,235)\nNon-Regulated\n(6,907)\n(5,982)\n(4,732)\nTotal cost of sales\n(10,069)\n(8,430)\n(6,967)\nOperating margin\n2,548 \n2,711 \n2,693 \nGeneral and administrative expenses\n(207)\n(166)\n(165)\nInterest expense\n(1,117)\n(911)\n(1,038)\nInterest income\n389 \n298 \n268 \nLoss on extinguishment of debt\n(15)\n(78)\n(186)\nOther expense\n(68)\n(60)\n(53)\nOther income\n102 \n410 \n75 \nLoss on disposal and sale of business interests\n(9)\n(1,683)\n(95)\nGoodwill impairment expense\n(777)\n \n \nAsset impairment expense\n(763)\n(1,575)\n(864)\nForeign currency transaction gains (losses)\n(77)\n(10)\n55 \nOther non-operating expense\n(175)\n \n(202)\nINCOME (LOSS) FROM CONTINUING OPERATIONS BEFORE TAXES AND EQUITY IN EARNINGS OF AFFILIATES\n(169)\n(1,064)\n488 \nIncome tax benefit (expense)\n(265)\n133 \n(216)\nNet equity in losses of affiliates\n(71)\n(24)\n(123)\nINCOME (LOSS) FROM CONTINUING OPERATIONS\n(505)\n(955)\n149 \nGain from disposal of discontinued businesses, net of income tax expense of $0, $1, and $0, respectively\n \n4 \n3 \nNET INCOME (LOSS)\n(505)\n(951)\n152 \nLess: Net loss (income) attributable to noncontrolling interests and redeemable stock of subsidiaries\n(41)\n542 \n(106)\nNET INCOME (LOSS) ATTRIBUTABLE TO THE AES CORPORATION\n$\n(546)\n$\n(409)\n$\n46 \nAMOUNTS ATTRIBUTABLE TO THE AES CORPORATION COMMON STOCKHOLDERS:\nIncome (loss) from continuing operations, net of tax\n$\n(546)\n$\n(413)\n$\n43 \nIncome from discontinued operations, net of tax\n \n4 \n3 \nNET INCOME (LOSS) ATTRIBUTABLE TO THE AES CORPORATION\n$\n(546)\n$\n(409)\n$\n46 \nBASIC EARNINGS PER SHARE:\nIncome (loss) from continuing operations attributable to The AES Corporation common stockholders, net of tax\n$\n(0.82)\n$\n(0.62)\n$\n0.06 \nIncome from discontinued operations attributable to The AES Corporation common stockholders, net of tax\n \n0.01 \n0.01 \nNET INCOME (LOSS) ATTRIBUTABLE TO THE AES CORPORATION COMMON STOCKHOLDERS\n$\n(0.82)\n$\n(0.61)\n$\n0.07 \nDILUTED EARNINGS PER SHARE:\nIncome (loss) from continuing operations attributable to The AES Corporation common stockholders, net of tax\n$\n(0.82)\n$\n(0.62)\n$\n0.06 \nIncome from discontinued operations attributable to The AES Corporation common stockholders, net of tax\n \n0.01 \n0.01 \nNET INCOME (LOSS) ATTRIBUTABLE TO THE AES CORPORATION COMMON STOCKHOLDERS\n$\n(0.82)\n$\n(0.61)\n$\n0.07 \nSee Accompanying Notes to Consolidated Financial Statements.\n"}]} +{"financebench_id": "financebench_id_10420", "company": "AES Corporation", "doc_name": "AES_2022_10K", "question_type": "metrics-generated", "question_reasoning": "Numerical reasoning", "domain_question_num": null, "question": "Based on the information provided primarily in the statement of financial position and the statement of income, what is AES's FY2022 return on assets (ROA)? ROA is defined as: FY2022 net income / (average total assets between FY2021 and FY2022). Round your answer to two decimal places.", "answer": "-0.02", "justification": "The metric in question was calculated using other simpler metrics. The various simpler metrics (from the current and, if relevant, previous fiscal year(s)) used were:\n\nMetric 1: Net income. This metric was located in the 10K as a single line item named: NET INCOME (LOSS) ATTRIBUTABLE TO THE AES CORPORATION.\n\nMetric 2: Total assets. This metric was located in the 10K as a single line item named: TOTAL ASSETS.", "dataset_subset_label": "OPEN_SOURCE", "evidence": [{"evidence_text": "128 \nConsolidated Balance Sheets\nDecember 31, 2022 and 2021\n2022\n2021\n(in millions, except share and per share data)\nASSETS\nCURRENT ASSETS\nCash and cash equivalents\n$\n1,374 \n$\n943 \nRestricted cash\n536 \n304 \nShort-term investments\n730 \n232 \nAccounts receivable, net of allowance for doubtful accounts of $5 and $5, respectively\n1,799 \n1,418 \nInventory\n1,055 \n604 \nPrepaid expenses\n98 \n142 \nOther current assets, net of CECL allowance of $2 and $0, respectively\n1,533 \n897 \nCurrent held-for-sale assets\n518 \n816 \nTotal current assets\n7,643 \n5,356 \nNONCURRENT ASSETS\nProperty, Plant and Equipment:\nLand\n470 \n426 \nElectric generation, distribution assets and other\n26,599 \n25,552 \nAccumulated depreciation\n(8,651)\n(8,486)\nConstruction in progress\n4,621 \n2,414 \nProperty, plant and equipment, net\n23,039 \n19,906 \nOther Assets:\nInvestments in and advances to affiliates\n952 \n1,080 \nDebt service reserves and other deposits\n177 \n237 \nGoodwill\n362 \n1,177 \nOther intangible assets, net of accumulated amortization of $434 and $385, respectively\n1,841 \n1,450 \nDeferred income taxes\n319 \n409 \nLoan receivable, net of allowance of $26\n1,051 \n \nOther noncurrent assets, net of allowance of $51 and $23, respectively\n2,979 \n2,188 \nNoncurrent held-for-sale assets\n \n1,160 \nTotal other assets\n7,681 \n7,701 \nTOTAL ASSETS\n$\n38,363 \n$\n32,963 \nLIABILITIES AND EQUITY\nCURRENT LIABILITIES\nAccounts payable\n$\n1,730 \n$\n1,153 \nAccrued interest\n249 \n182 \nAccrued non-income taxes\n249 \n266 \nAccrued and other liabilities\n2,151 \n1,205 \nNon-recourse debt, including $416 and $302, respectively, related to variable interest entities\n1,758 \n1,367 \nCurrent held-for-sale liabilities\n354 \n559 \nTotal current liabilities\n6,491 \n4,732 \nNONCURRENT LIABILITIES\nRecourse debt\n3,894 \n3,729 \nNon-recourse debt, including $2,295 and $2,223, respectively, related to variable interest entities\n17,846 \n13,603 \nDeferred income taxes\n1,139 \n977 \nOther noncurrent liabilities\n3,168 \n3,358 \nNoncurrent held-for-sale liabilities\n \n740 \nTotal noncurrent liabilities\n26,047 \n22,407 \nCommitments and Contingencies (see Notes 12 and 13)\nRedeemable stock of subsidiaries\n1,321 \n1,257 \nEQUITY\nTHE AES CORPORATION STOCKHOLDERS EQUITY\nPreferred stock (without par value, 50,000,000 shares authorized; 1,043,050 issued and outstanding at December 31, 2022 and\nDecember 31, 2021)\n838 \n838 \nCommon stock ($0.01 par value, 1,200,000,000 shares authorized; 818,790,001 issued and 668,743,464 outstanding at December\n31, 2022 and 818,717,043 issued and 666,793,625 outstanding at December 31, 2021)\n8 \n8 \nAdditional paid-in capital\n6,688 \n7,106 \nAccumulated deficit\n(1,635)\n(1,089)\nAccumulated other comprehensive loss\n(1,640)\n(2,220)\nTreasury stock, at cost (150,046,537 and 151,923,418 shares at December 31, 2022 and December 31, 2021, respectively)\n(1,822)\n(1,845)\nTotal AES Corporation stockholders equity\n2,437 \n2,798 \nNONCONTROLLING INTERESTS\n2,067 \n1,769 \nTotal equity\n4,504 \n4,567 \nTOTAL LIABILITIES AND EQUITY\n$\n38,363 \n$\n32,963 \nSee Accompanying Notes to Consolidated Financial Statements.", "doc_name": "AES_2022_10K", "evidence_page_num": 129, "evidence_text_full_page": "128 \nConsolidated Balance Sheets\nDecember 31, 2022 and 2021\n2022\n2021\n(in millions, except share and per share data)\nASSETS\nCURRENT ASSETS\nCash and cash equivalents\n$\n1,374 \n$\n943 \nRestricted cash\n536 \n304 \nShort-term investments\n730 \n232 \nAccounts receivable, net of allowance for doubtful accounts of $5 and $5, respectively\n1,799 \n1,418 \nInventory\n1,055 \n604 \nPrepaid expenses\n98 \n142 \nOther current assets, net of CECL allowance of $2 and $0, respectively\n1,533 \n897 \nCurrent held-for-sale assets\n518 \n816 \nTotal current assets\n7,643 \n5,356 \nNONCURRENT ASSETS\nProperty, Plant and Equipment:\nLand\n470 \n426 \nElectric generation, distribution assets and other\n26,599 \n25,552 \nAccumulated depreciation\n(8,651)\n(8,486)\nConstruction in progress\n4,621 \n2,414 \nProperty, plant and equipment, net\n23,039 \n19,906 \nOther Assets:\nInvestments in and advances to affiliates\n952 \n1,080 \nDebt service reserves and other deposits\n177 \n237 \nGoodwill\n362 \n1,177 \nOther intangible assets, net of accumulated amortization of $434 and $385, respectively\n1,841 \n1,450 \nDeferred income taxes\n319 \n409 \nLoan receivable, net of allowance of $26\n1,051 \n \nOther noncurrent assets, net of allowance of $51 and $23, respectively\n2,979 \n2,188 \nNoncurrent held-for-sale assets\n \n1,160 \nTotal other assets\n7,681 \n7,701 \nTOTAL ASSETS\n$\n38,363 \n$\n32,963 \nLIABILITIES AND EQUITY\nCURRENT LIABILITIES\nAccounts payable\n$\n1,730 \n$\n1,153 \nAccrued interest\n249 \n182 \nAccrued non-income taxes\n249 \n266 \nAccrued and other liabilities\n2,151 \n1,205 \nNon-recourse debt, including $416 and $302, respectively, related to variable interest entities\n1,758 \n1,367 \nCurrent held-for-sale liabilities\n354 \n559 \nTotal current liabilities\n6,491 \n4,732 \nNONCURRENT LIABILITIES\nRecourse debt\n3,894 \n3,729 \nNon-recourse debt, including $2,295 and $2,223, respectively, related to variable interest entities\n17,846 \n13,603 \nDeferred income taxes\n1,139 \n977 \nOther noncurrent liabilities\n3,168 \n3,358 \nNoncurrent held-for-sale liabilities\n \n740 \nTotal noncurrent liabilities\n26,047 \n22,407 \nCommitments and Contingencies (see Notes 12 and 13)\nRedeemable stock of subsidiaries\n1,321 \n1,257 \nEQUITY\nTHE AES CORPORATION STOCKHOLDERS EQUITY\nPreferred stock (without par value, 50,000,000 shares authorized; 1,043,050 issued and outstanding at December 31, 2022 and\nDecember 31, 2021)\n838 \n838 \nCommon stock ($0.01 par value, 1,200,000,000 shares authorized; 818,790,001 issued and 668,743,464 outstanding at December\n31, 2022 and 818,717,043 issued and 666,793,625 outstanding at December 31, 2021)\n8 \n8 \nAdditional paid-in capital\n6,688 \n7,106 \nAccumulated deficit\n(1,635)\n(1,089)\nAccumulated other comprehensive loss\n(1,640)\n(2,220)\nTreasury stock, at cost (150,046,537 and 151,923,418 shares at December 31, 2022 and December 31, 2021, respectively)\n(1,822)\n(1,845)\nTotal AES Corporation stockholders equity\n2,437 \n2,798 \nNONCONTROLLING INTERESTS\n2,067 \n1,769 \nTotal equity\n4,504 \n4,567 \nTOTAL LIABILITIES AND EQUITY\n$\n38,363 \n$\n32,963 \nSee Accompanying Notes to Consolidated Financial Statements.\n"}, {"evidence_text": "129 \nConsolidated Statements of Operations\nYears ended December 31, 2022, 2021, and 2020\n2022\n2021\n2020\n(in millions, except per share amounts)\nRevenue:\nRegulated\n$\n3,538 \n$\n2,868 \n$\n2,661 \nNon-Regulated\n9,079 \n8,273 \n6,999 \nTotal revenue\n12,617 \n11,141 \n9,660 \nCost of Sales:\nRegulated\n(3,162)\n(2,448)\n(2,235)\nNon-Regulated\n(6,907)\n(5,982)\n(4,732)\nTotal cost of sales\n(10,069)\n(8,430)\n(6,967)\nOperating margin\n2,548 \n2,711 \n2,693 \nGeneral and administrative expenses\n(207)\n(166)\n(165)\nInterest expense\n(1,117)\n(911)\n(1,038)\nInterest income\n389 \n298 \n268 \nLoss on extinguishment of debt\n(15)\n(78)\n(186)\nOther expense\n(68)\n(60)\n(53)\nOther income\n102 \n410 \n75 \nLoss on disposal and sale of business interests\n(9)\n(1,683)\n(95)\nGoodwill impairment expense\n(777)\n \n \nAsset impairment expense\n(763)\n(1,575)\n(864)\nForeign currency transaction gains (losses)\n(77)\n(10)\n55 \nOther non-operating expense\n(175)\n \n(202)\nINCOME (LOSS) FROM CONTINUING OPERATIONS BEFORE TAXES AND EQUITY IN EARNINGS OF AFFILIATES\n(169)\n(1,064)\n488 \nIncome tax benefit (expense)\n(265)\n133 \n(216)\nNet equity in losses of affiliates\n(71)\n(24)\n(123)\nINCOME (LOSS) FROM CONTINUING OPERATIONS\n(505)\n(955)\n149 \nGain from disposal of discontinued businesses, net of income tax expense of $0, $1, and $0, respectively\n \n4 \n3 \nNET INCOME (LOSS)\n(505)\n(951)\n152 \nLess: Net loss (income) attributable to noncontrolling interests and redeemable stock of subsidiaries\n(41)\n542 \n(106)\nNET INCOME (LOSS) ATTRIBUTABLE TO THE AES CORPORATION\n$\n(546)\n$\n(409)\n$\n46 \nAMOUNTS ATTRIBUTABLE TO THE AES CORPORATION COMMON STOCKHOLDERS:\nIncome (loss) from continuing operations, net of tax\n$\n(546)\n$\n(413)\n$\n43 \nIncome from discontinued operations, net of tax\n \n4 \n3 \nNET INCOME (LOSS) ATTRIBUTABLE TO THE AES CORPORATION\n$\n(546)\n$\n(409)\n$\n46 \nBASIC EARNINGS PER SHARE:\nIncome (loss) from continuing operations attributable to The AES Corporation common stockholders, net of tax\n$\n(0.82)\n$\n(0.62)\n$\n0.06 \nIncome from discontinued operations attributable to The AES Corporation common stockholders, net of tax\n \n0.01 \n0.01 \nNET INCOME (LOSS) ATTRIBUTABLE TO THE AES CORPORATION COMMON STOCKHOLDERS\n$\n(0.82)\n$\n(0.61)\n$\n0.07 \nDILUTED EARNINGS PER SHARE:\nIncome (loss) from continuing operations attributable to The AES Corporation common stockholders, net of tax\n$\n(0.82)\n$\n(0.62)\n$\n0.06 \nIncome from discontinued operations attributable to The AES Corporation common stockholders, net of tax\n \n0.01 \n0.01 \nNET INCOME (LOSS) ATTRIBUTABLE TO THE AES CORPORATION COMMON STOCKHOLDERS\n$\n(0.82)\n$\n(0.61)\n$\n0.07 \nSee Accompanying Notes to Consolidated Financial Statements.", "doc_name": "AES_2022_10K", "evidence_page_num": 131, "evidence_text_full_page": "129 \nConsolidated Statements of Operations\nYears ended December 31, 2022, 2021, and 2020\n2022\n2021\n2020\n(in millions, except per share amounts)\nRevenue:\nRegulated\n$\n3,538 \n$\n2,868 \n$\n2,661 \nNon-Regulated\n9,079 \n8,273 \n6,999 \nTotal revenue\n12,617 \n11,141 \n9,660 \nCost of Sales:\nRegulated\n(3,162)\n(2,448)\n(2,235)\nNon-Regulated\n(6,907)\n(5,982)\n(4,732)\nTotal cost of sales\n(10,069)\n(8,430)\n(6,967)\nOperating margin\n2,548 \n2,711 \n2,693 \nGeneral and administrative expenses\n(207)\n(166)\n(165)\nInterest expense\n(1,117)\n(911)\n(1,038)\nInterest income\n389 \n298 \n268 \nLoss on extinguishment of debt\n(15)\n(78)\n(186)\nOther expense\n(68)\n(60)\n(53)\nOther income\n102 \n410 \n75 \nLoss on disposal and sale of business interests\n(9)\n(1,683)\n(95)\nGoodwill impairment expense\n(777)\n \n \nAsset impairment expense\n(763)\n(1,575)\n(864)\nForeign currency transaction gains (losses)\n(77)\n(10)\n55 \nOther non-operating expense\n(175)\n \n(202)\nINCOME (LOSS) FROM CONTINUING OPERATIONS BEFORE TAXES AND EQUITY IN EARNINGS OF AFFILIATES\n(169)\n(1,064)\n488 \nIncome tax benefit (expense)\n(265)\n133 \n(216)\nNet equity in losses of affiliates\n(71)\n(24)\n(123)\nINCOME (LOSS) FROM CONTINUING OPERATIONS\n(505)\n(955)\n149 \nGain from disposal of discontinued businesses, net of income tax expense of $0, $1, and $0, respectively\n \n4 \n3 \nNET INCOME (LOSS)\n(505)\n(951)\n152 \nLess: Net loss (income) attributable to noncontrolling interests and redeemable stock of subsidiaries\n(41)\n542 \n(106)\nNET INCOME (LOSS) ATTRIBUTABLE TO THE AES CORPORATION\n$\n(546)\n$\n(409)\n$\n46 \nAMOUNTS ATTRIBUTABLE TO THE AES CORPORATION COMMON STOCKHOLDERS:\nIncome (loss) from continuing operations, net of tax\n$\n(546)\n$\n(413)\n$\n43 \nIncome from discontinued operations, net of tax\n \n4 \n3 \nNET INCOME (LOSS) ATTRIBUTABLE TO THE AES CORPORATION\n$\n(546)\n$\n(409)\n$\n46 \nBASIC EARNINGS PER SHARE:\nIncome (loss) from continuing operations attributable to The AES Corporation common stockholders, net of tax\n$\n(0.82)\n$\n(0.62)\n$\n0.06 \nIncome from discontinued operations attributable to The AES Corporation common stockholders, net of tax\n \n0.01 \n0.01 \nNET INCOME (LOSS) ATTRIBUTABLE TO THE AES CORPORATION COMMON STOCKHOLDERS\n$\n(0.82)\n$\n(0.61)\n$\n0.07 \nDILUTED EARNINGS PER SHARE:\nIncome (loss) from continuing operations attributable to The AES Corporation common stockholders, net of tax\n$\n(0.82)\n$\n(0.62)\n$\n0.06 \nIncome from discontinued operations attributable to The AES Corporation common stockholders, net of tax\n \n0.01 \n0.01 \nNET INCOME (LOSS) ATTRIBUTABLE TO THE AES CORPORATION COMMON STOCKHOLDERS\n$\n(0.82)\n$\n(0.61)\n$\n0.07 \nSee Accompanying Notes to Consolidated Financial Statements.\n"}]} +{"financebench_id": "financebench_id_00685", "company": "Best Buy", "doc_name": "BESTBUY_2023_10K", "question_type": "domain-relevant", "question_reasoning": "Logical reasoning (based on numerical reasoning) OR Logical reasoning", "domain_question_num": "dg03", "question": "Are Best Buy's gross margins historically consistent (not fluctuating more than roughly 2% each year)? If gross margins are not a relevant metric for a company like this, then please state that and explain why.", "answer": "Yes, the margins have been consistent, there has been a minor decline of 1.1% in gross margins between FY2022 and FY2023.", "justification": "Gross Profit/Revenue\n9912/46298\n11640/51761", "dataset_subset_label": "OPEN_SOURCE", "evidence": [{"evidence_text": "Consolidated Statements of Earnings\n$ and shares in millions, except per share amounts\n \n \n \n \n \n \n \n \n \n \n \n \n \nFiscal Years Ended\nJanuary 28, 2023\n \nJanuary 29, 2022\n \nJanuary 30, 2021\nRevenue\n$\n 46,298 \n \n$\n 51,761 \n \n$\n 47,262 \nCost of sales\n \n 36,386 \n \n \n 40,121 \n \n \n 36,689 \nGross profit\n \n 9,912 \n \n \n 11,640 \n \n \n 10,573 \nSelling, general and administrative expenses\n \n 7,970 \n \n \n 8,635 \n \n \n 7,928 \nRestructuring charges\n \n 147 \n \n \n (34) \n \n \n 254 \nOperating income\n \n 1,795 \n \n \n 3,039 \n \n \n 2,391 \nOther income (expense):\n \n \n \n \n \n \n \n \n \n \n \nInvestment income and other\n \n 28 \n \n \n 10 \n \n \n 38 \nInterest expense\n \n (35) \n \n \n (25) \n \n \n (52) \nEarnings before income tax expense and equity in income of affiliates\n \n 1,788 \n \n \n 3,024 \n \n \n 2,377 \nIncome tax expense\n \n 370 \n \n \n 574 \n \n \n 579 \nEquity in income of affiliates\n \n 1 \n \n \n 4 \n \n \n - \nNet earnings\n$\n 1,419 \n \n$\n 2,454 \n \n$\n 1,798", "doc_name": "BESTBUY_2023_10K", "evidence_page_num": 39, "evidence_text_full_page": " \nConsolidated Statements of Earnings\n$ and shares in millions, except per share amounts\n \n \n \n \n \n \n \n \n \n \n \n \n \nFiscal Years Ended\nJanuary 28, 2023\n \nJanuary 29, 2022\n \nJanuary 30, 2021\nRevenue\n$\n 46,298 \n \n$\n 51,761 \n \n$\n 47,262 \nCost of sales\n \n 36,386 \n \n \n 40,121 \n \n \n 36,689 \nGross profit\n \n 9,912 \n \n \n 11,640 \n \n \n 10,573 \nSelling, general and administrative expenses\n \n 7,970 \n \n \n 8,635 \n \n \n 7,928 \nRestructuring charges\n \n 147 \n \n \n (34) \n \n \n 254 \nOperating income\n \n 1,795 \n \n \n 3,039 \n \n \n 2,391 \nOther income (expense):\n \n \n \n \n \n \n \n \n \n \n \nInvestment income and other\n \n 28 \n \n \n 10 \n \n \n 38 \nInterest expense\n \n (35) \n \n \n (25) \n \n \n (52) \nEarnings before income tax expense and equity in income of affiliates\n \n 1,788 \n \n \n 3,024 \n \n \n 2,377 \nIncome tax expense\n \n 370 \n \n \n 574 \n \n \n 579 \nEquity in income of affiliates\n \n 1 \n \n \n 4 \n \n \n - \nNet earnings\n$\n 1,419 \n \n$\n 2,454 \n \n$\n 1,798 \n \n \n \n \n \n \n \n \n \n \n \n \nBasic earnings per share\n$\n 6.31 \n \n$\n 9.94 \n \n$\n 6.93 \nDiluted earnings per share\n$\n 6.29 \n \n$\n 9.84 \n \n$\n 6.84 \n \n \n \n \n \n \n \n \n \n \n \n \nWeighted-average common shares outstanding:\n \n \n \n \n \n \n \n \n \n \n \nBasic\n \n 224.8 \n \n \n 246.8 \n \n \n 259.6 \nDiluted\n \n 225.7 \n \n \n 249.3 \n \n \n 263.0 \n \nSee Notes to Consolidated Financial Statements.\n \n \n40\n"}]} +{"financebench_id": "financebench_id_01077", "company": "Best Buy", "doc_name": "BESTBUY_2023_10K", "question_type": "domain-relevant", "question_reasoning": "Information extraction", "domain_question_num": "dg10", "question": "What are major acquisitions that Best Buy has done in FY2023, FY2022 and FY2021?", "answer": "Best Buy closed two acquisitions, both these companies were already partially owned by Best Buy, but Best Buy acquired all outstanding shares of these two companies during FY 2022: (1) Current Health Ltd and (2) Two Peaks, LLC d/b/a Yardbird Furniture", "justification": null, "dataset_subset_label": "OPEN_SOURCE", "evidence": [{"evidence_text": "Acquisitions\n \nCurrent Health Ltd.\n \nIn fiscal 2022, we acquired all of the outstanding shares of Current Health Ltd. (Current Health), a care-at-home technology platform, on November 2, 2021, for \nnet cash consideration of $389 million. The acquired assets included $351 million of goodwill that was assigned to our Best Buy Health reporting unit and was \ndeductible for income tax purposes. The acquisition is aligned with our focus in virtual care to enable people in their homes to connect seamlessly with their \nhealth care providers and is included in our Domestic reportable segment and Services revenue category. The acquisition was accounted for using the \nacquisition method of accounting for business combinations and was not material to the results of operations.\n \nTwo Peaks, LLC d/b/a Yardbird Furniture\n \nIn fiscal 2022, we acquired all of the outstanding shares of Two Peaks, LLC d/b/a Yardbird Furniture (Yardbird), a direct-to-consumer outdoor furniture company, \non November 4, 2021, for net cash consideration of $79 million. The acquired assets included $47 million of goodwill that was assigned to our Best Buy Domestic \nreporting unit and was deductible for income tax purposes. The acquisition expands our assortment in categories like outdoor living, as more and more \nconsumers look to make over or upgrade their outdoor living spaces. The acquisition was accounted for using the acquisition method of accounting for business \ncombinations and was not material to the results of our operations.", "doc_name": "BESTBUY_2023_10K", "evidence_page_num": 50, "evidence_text_full_page": "Vendor Allowances\n \nWe receive funds from our merchandise vendors through a variety of programs and arrangements, primarily in the form of purchases-based or sales-based \nvolumes and for product advertising and placement. We recognize allowances based on purchases and sales as a reduction of cost of sales when the associated \ninventory is sold. Allowances for advertising and placement are recognized as a reduction of cost of sales ratably over the corresponding performance period. \nFunds that are determined to be a reimbursement of specific, incremental and identifiable costs incurred to sell a vendors products are recorded as an offset to \nthe related expense within SG&A when incurred.\n \nAdvertising Costs\n \nAdvertising costs, which are included in SG&A, are expensed the first time the advertisement runs. Advertising costs consist primarily of digital advertisements. \nAdvertising expenses were $864 million, $915 million and $819 million in fiscal 2023, fiscal 2022 and fiscal 2021, respectively.\n \nStock-Based Compensation\n \nWe recognize stock-based compensation expense for the fair value of our stock-based compensation awards, which is determined based on the closing market \nprice of our stock at the date of grant for time-based and performance-based share awards, and Monte-Carlo simulation for market-based share awards. \nCompensation expense is recognized on a straight-line basis over the period in which services are required, except for performance-based share awards that \nvest on a graded basis, in which case the expense is front-loaded or recognized on a graded-attribution basis. Forfeitures are expensed as incurred or upon \ntermination.\n \nComprehensive Income (Loss)\n \nComprehensive income (loss) is computed as net earnings plus certain other items that are recorded directly to shareholders equity.\n \n2. Acquisitions\n \nCurrent Health Ltd.\n \nIn fiscal 2022, we acquired all of the outstanding shares of Current Health Ltd. (Current Health), a care-at-home technology platform, on November 2, 2021, for \nnet cash consideration of $389 million. The acquired assets included $351 million of goodwill that was assigned to our Best Buy Health reporting unit and was \ndeductible for income tax purposes. The acquisition is aligned with our focus in virtual care to enable people in their homes to connect seamlessly with their \nhealth care providers and is included in our Domestic reportable segment and Services revenue category. The acquisition was accounted for using the \nacquisition method of accounting for business combinations and was not material to the results of operations.\n \nTwo Peaks, LLC d/b/a Yardbird Furniture\n \nIn fiscal 2022, we acquired all of the outstanding shares of Two Peaks, LLC d/b/a Yardbird Furniture (Yardbird), a direct-to-consumer outdoor furniture company, \non November 4, 2021, for net cash consideration of $79 million. The acquired assets included $47 million of goodwill that was assigned to our Best Buy Domestic \nreporting unit and was deductible for income tax purposes. The acquisition expands our assortment in categories like outdoor living, as more and more \nconsumers look to make over or upgrade their outdoor living spaces. The acquisition was accounted for using the acquisition method of accounting for business \ncombinations and was not material to the results of our operations.\n \n3. Restructuring\n \n \nRestructuring charges were as follows ($ in millions):\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n2023\n \n2022\n \n2021\nFiscal 2023 Resource Optimization Initiative\n \n \n$\n 145 \n \n$\n - \n \n$\n - \nMexico Exit and Strategic Realignment(1)\n \n \n \n 2 \n \n \n (41) \n \n \n 277 \nFiscal 2020 U.S. Retail Operating Model Changes\n \n -\n \n \n 1 \n \n \n -\n \nTotal\n \n \n \n \n \n$\n 147 \n \n$\n (40) \n \n$\n 277 \n \n(1)\nIncludes ($6) million and $23 million related to inventory markdowns recorded in Cost of sales on our Consolidated Statements of Earnings in fiscal 2022 and fiscal 2021, respectively.\n \nFiscal 2023 Resource Optimization Initiative\n \nIn light of ongoing changes in business trends, during the second quarter of fiscal 2023, we commenced an enterprise-wide initiative to better align our spending \nwith critical strategies and operations, as well as to optimize our cost structure. Charges incurred relate to employee termination benefits within our Domestic and \nInternational segments of $140 million and $5 million, respectively. We currently do not expect the remaining charges in fiscal 2024 related to this initiative to be \nmaterial to the results of our operations.\n \nAll charges incurred related to this initiative were from continuing operations and were presented within Restructuring charges on our Consolidated Statements of \nEarnings. \n \n51\n"}]} +{"financebench_id": "financebench_id_01275", "company": "Best Buy", "doc_name": "BESTBUY_2023_10K", "question_type": "domain-relevant", "question_reasoning": "Numerical reasoning", "domain_question_num": "dg19", "question": "Among operations, investing, and financing activities, which brought in the most (or lost the least) cash flow for Best Buy in FY2023?", "answer": "Best Buy generated the most cash flow from operating activities in FY 2023 ($1.8 bn)", "justification": null, "dataset_subset_label": "OPEN_SOURCE", "evidence": [{"evidence_text": "Consolidated Statements of Cash Flows\n$ in millions\n \n \n \n \n \n \n \n \n \n \n \n \n \n \nFiscal Years Ended\nJanuary 28, 2023\n \nJanuary 29, 2022\n \nJanuary 30, 2021\nOperating activities\n \n \n \n \n \n \n \n \n \n \n \nNet earnings\n$\n 1,419 \n \n$\n 2,454 \n \n$\n 1,798 \n \nAdjustments to reconcile net earnings to total cash provided by operating activities:\n \n \n \n \n \n \n \n \nDepreciation and amortization\n \n 918 \n \n \n 869 \n \n \n 839 \nRestructuring charges\n \n 147 \n \n \n (34) \n \n \n 254 \nStock-based compensation\n \n 138 \n \n \n 141 \n \n \n 135 \nDeferred income taxes\n \n 51 \n \n \n 14 \n \n \n (36) \nOther, net\n \n 12 \n \n \n 11 \n \n \n 3 \nChanges in operating assets and liabilities, net of acquired assets and liabilities:\n \n \n \n \n \n \n \n \n \nReceivables\n \n (103) \n \n \n 17 \n \n \n 73 \nMerchandise inventories\n \n 809 \n \n \n (328) \n \n \n (435) \nOther assets\n \n (21) \n \n \n (14) \n \n \n (51) \nAccounts payable\n \n (1,099) \n \n \n (201) \n \n \n 1,676 \nIncome taxes\n \n 36 \n \n \n (156) \n \n \n 173 \nOther liabilities\n \n (483) \n \n \n 479 \n \n \n 498 \nTotal cash provided by operating activities\n \n 1,824 \n \n \n 3,252 \n \n \n 4,927 \nInvesting activities\n \n \n \n \n \n \n \n \n \n \n \nAdditions to property and equipment, net of $35, $46 and $32, respectively, of non-cash \ncapital expenditures\n \n (930) \n \n \n (737) \n \n \n (713) \nPurchases of investments\n \n (46) \n \n \n (233) \n \n \n (620) \nSales of investments\n \n 7 \n \n \n 66 \n \n \n 546 \nAcquisitions, net of cash acquired\n \n - \n \n \n (468) \n \n \n - \nOther, net\n \n 7 \n \n \n - \n \n \n (1) \nTotal cash used in investing activities\n \n (962) \n \n \n (1,372) \n \n \n (788) \nFinancing activities\n \n \n \n \n \n \n \n \n \n \n \nRepurchase of common stock\n \n (1,014) \n \n \n (3,502) \n \n \n (312) \nIssuance of common stock\n \n 16 \n \n \n 29 \n \n \n 28 \nDividends paid\n \n (789) \n \n \n (688) \n \n \n (568) \nBorrowings of debt\n \n - \n \n \n - \n \n \n 1,892 \nRepayments of debt\n \n (19) \n \n \n (133) \n \n \n (1,916) \nOther, net\n \n - \n \n \n (3) \n \n \n - \nTotal cash used in financing activities\n \n (1,806) \n \n \n (4,297) \n \n \n (876) \n \n \n \n \n \n \n \n \n \n \n \n \nEffect of exchange rate changes on cash\n \n (8) \n \n \n (3) \n \n \n 7 \nIncrease (decrease) in cash, cash equivalents and restricted cash\n \n (952) \n \n \n (2,420) \n \n \n 3,270 \nCash, cash equivalents and restricted cash at beginning of period\n \n 3,205 \n \n \n 5,625 \n \n \n 2,355 \nCash, cash equivalents and restricted cash at end of period\n$\n 2,253 \n \n$\n 3,205 \n \n$\n 5,625", "doc_name": "BESTBUY_2023_10K", "evidence_page_num": 41, "evidence_text_full_page": " \nConsolidated Statements of Cash Flows\n$ in millions\n \n \n \n \n \n \n \n \n \n \n \n \n \n \nFiscal Years Ended\nJanuary 28, 2023\n \nJanuary 29, 2022\n \nJanuary 30, 2021\nOperating activities\n \n \n \n \n \n \n \n \n \n \n \nNet earnings\n$\n 1,419 \n \n$\n 2,454 \n \n$\n 1,798 \n \nAdjustments to reconcile net earnings to total cash provided by operating activities:\n \n \n \n \n \n \n \n \nDepreciation and amortization\n \n 918 \n \n \n 869 \n \n \n 839 \nRestructuring charges\n \n 147 \n \n \n (34) \n \n \n 254 \nStock-based compensation\n \n 138 \n \n \n 141 \n \n \n 135 \nDeferred income taxes\n \n 51 \n \n \n 14 \n \n \n (36) \nOther, net\n \n 12 \n \n \n 11 \n \n \n 3 \nChanges in operating assets and liabilities, net of acquired assets and liabilities:\n \n \n \n \n \n \n \n \n \nReceivables\n \n (103) \n \n \n 17 \n \n \n 73 \nMerchandise inventories\n \n 809 \n \n \n (328) \n \n \n (435) \nOther assets\n \n (21) \n \n \n (14) \n \n \n (51) \nAccounts payable\n \n (1,099) \n \n \n (201) \n \n \n 1,676 \nIncome taxes\n \n 36 \n \n \n (156) \n \n \n 173 \nOther liabilities\n \n (483) \n \n \n 479 \n \n \n 498 \nTotal cash provided by operating activities\n \n 1,824 \n \n \n 3,252 \n \n \n 4,927 \nInvesting activities\n \n \n \n \n \n \n \n \n \n \n \nAdditions to property and equipment, net of $35, $46 and $32, respectively, of non-cash \ncapital expenditures\n \n (930) \n \n \n (737) \n \n \n (713) \nPurchases of investments\n \n (46) \n \n \n (233) \n \n \n (620) \nSales of investments\n \n 7 \n \n \n 66 \n \n \n 546 \nAcquisitions, net of cash acquired\n \n - \n \n \n (468) \n \n \n - \nOther, net\n \n 7 \n \n \n - \n \n \n (1) \nTotal cash used in investing activities\n \n (962) \n \n \n (1,372) \n \n \n (788) \nFinancing activities\n \n \n \n \n \n \n \n \n \n \n \nRepurchase of common stock\n \n (1,014) \n \n \n (3,502) \n \n \n (312) \nIssuance of common stock\n \n 16 \n \n \n 29 \n \n \n 28 \nDividends paid\n \n (789) \n \n \n (688) \n \n \n (568) \nBorrowings of debt\n \n - \n \n \n - \n \n \n 1,892 \nRepayments of debt\n \n (19) \n \n \n (133) \n \n \n (1,916) \nOther, net\n \n - \n \n \n (3) \n \n \n - \nTotal cash used in financing activities\n \n (1,806) \n \n \n (4,297) \n \n \n (876) \n \n \n \n \n \n \n \n \n \n \n \n \nEffect of exchange rate changes on cash\n \n (8) \n \n \n (3) \n \n \n 7 \nIncrease (decrease) in cash, cash equivalents and restricted cash\n \n (952) \n \n \n (2,420) \n \n \n 3,270 \nCash, cash equivalents and restricted cash at beginning of period\n \n 3,205 \n \n \n 5,625 \n \n \n 2,355 \nCash, cash equivalents and restricted cash at end of period\n$\n 2,253 \n \n$\n 3,205 \n \n$\n 5,625 \n \n \n \n \n \n \n \n \n \n \n \n \n \nSupplemental cash flow information\n \n \n \n \n \n \n \n \n \n \n \nIncome taxes paid\n$\n 283 \n \n$\n 716 \n \n$\n 442 \nInterest paid\n$\n 31 \n \n$\n 22 \n \n$\n 50 \n \nSee Notes to Consolidated Financial Statements.\n \n \n42\n"}]} +{"financebench_id": "financebench_id_01488", "company": "Johnson & Johnson", "doc_name": "JOHNSON_JOHNSON_2023_8K_dated-2023-08-30", "question_type": "novel-generated", "question_reasoning": null, "domain_question_num": null, "question": "Which business segment of JnJ will be treated as a discontinued operation from August 30, 2023 onward?", "answer": "The Consumer Health business segment will be treated as a discontinued operation from August 30, 2023 onward.", "justification": null, "dataset_subset_label": "OPEN_SOURCE", "evidence": [{"evidence_text": "Exhibit 99.1\nJohnson & Johnson Announces Updated Financials and 2023 Guidance Following Completion of the Kenvue\nSeparation\n\nCompany expects increased 2023 Reported Sales Growth of 7.0% - 8.0%, Operational Sales Growth of 7.5% - 8.5%, and\nAdjusted Operational Sales Growth of 6.2% - 7.2%; Figures exclude the COVID-19 Vaccine\n\nCompany expects 2023 Adjusted Reported Earnings Per Share (EPS) of $10.00 - $10.10, reflecting increased growth of\n12.5% at the mid-point and Adjusted Operational EPS of $9.90 - $10.00, reflecting increased growth of 11.5% at the mid-\npoint\n\nCompany reduced outstanding share count by approximately 191 million; 2023 guidance reflects only a partial-year benefit\nof approximately 73.5 million shares or $0.28 benefit to EPS\n\nCompany secured $13.2 billion in cash proceeds from the Kenvue debt offering and initial public offering and maintains 9.5%\nof equity stake in Kenvue\n\nCompany maintains its quarterly dividend of $1.19 per share\nNew Brunswick, N.J. (August 30, 2023) Johnson & Johnson (NYSE: JNJ) (the Company) today announced updates to its financials and\n2023 guidance which reflect its operations as a company focused on transformational innovation in Pharmaceutical and MedTech. The\nCompany has published a recorded webinar for investors to provide additional context behind the updated financials and 2023 guidance\nfound in this release, which may be accessed by visiting the Investors section of the Company's website at webcasts & presentations.\nThe completion of this transaction uniquely positions Johnson & Johnson as a Pharmaceutical and MedTech company focused on delivering\ntransformative healthcare solutions to patients, said Joaquin Duato, Chairman of the Board and Chief Executive Officer. We are incredibly\nproud of the focus and dedication of our employees worldwide to achieve this milestone, which we are confident will unlock near- and long-\nterm value for all of our stakeholders.\nAs previously announced, the Company recently completed an exchange offer to finalize the separation of Kenvue Inc., formerly Johnson &\nJohnsons Consumer Health business. As a result of the completion of the exchange offer, Johnson & Johnson will now present its\nConsumer Health business financial results as discontinued operations, including a gain of approximately $20 billion in the third quarter of\n2023", "doc_name": "JOHNSON_JOHNSON_2023_8K_dated-2023-08-30", "evidence_page_num": 3, "evidence_text_full_page": "Exhibit 99.1\nJohnson & Johnson Announces Updated Financials and 2023 Guidance Following Completion of the Kenvue\nSeparation\n\nCompany expects increased 2023 Reported Sales Growth of 7.0% - 8.0%, Operational Sales Growth of 7.5% - 8.5%, and\nAdjusted Operational Sales Growth of 6.2% - 7.2%; Figures exclude the COVID-19 Vaccine\n\nCompany expects 2023 Adjusted Reported Earnings Per Share (EPS) of $10.00 - $10.10, reflecting increased growth of\n12.5% at the mid-point and Adjusted Operational EPS of $9.90 - $10.00, reflecting increased growth of 11.5% at the mid-\npoint\n\nCompany reduced outstanding share count by approximately 191 million; 2023 guidance reflects only a partial-year benefit\nof approximately 73.5 million shares or $0.28 benefit to EPS\n\nCompany secured $13.2 billion in cash proceeds from the Kenvue debt offering and initial public offering and maintains 9.5%\nof equity stake in Kenvue\n\nCompany maintains its quarterly dividend of $1.19 per share\nNew Brunswick, N.J. (August 30, 2023) Johnson & Johnson (NYSE: JNJ) (the Company) today announced updates to its financials and\n2023 guidance which reflect its operations as a company focused on transformational innovation in Pharmaceutical and MedTech. The\nCompany has published a recorded webinar for investors to provide additional context behind the updated financials and 2023 guidance\nfound in this release, which may be accessed by visiting the Investors section of the Company's website at webcasts & presentations.\nThe completion of this transaction uniquely positions Johnson & Johnson as a Pharmaceutical and MedTech company focused on delivering\ntransformative healthcare solutions to patients, said Joaquin Duato, Chairman of the Board and Chief Executive Officer. We are incredibly\nproud of the focus and dedication of our employees worldwide to achieve this milestone, which we are confident will unlock near- and long-\nterm value for all of our stakeholders.\nAs previously announced, the Company recently completed an exchange offer to finalize the separation of Kenvue Inc., formerly Johnson &\nJohnsons Consumer Health business. As a result of the completion of the exchange offer, Johnson & Johnson will now present its\nConsumer Health business financial results as discontinued operations, including a gain of approximately $20 billion in the third quarter of\n2023.\n"}]} +{"financebench_id": "financebench_id_01490", "company": "Johnson & Johnson", "doc_name": "JOHNSON_JOHNSON_2023_8K_dated-2023-08-30", "question_type": "novel-generated", "question_reasoning": null, "domain_question_num": null, "question": "What is the amount of the gain accruing to JnJ as a result of the separation of its Consumer Health business segment, as of August 30, 2023?", "answer": "JnJ will make a gain of approximately $20 billion from the separation of its Consumer Health business segment.", "justification": null, "dataset_subset_label": "OPEN_SOURCE", "evidence": [{"evidence_text": "Exhibit 99.1\nJohnson & Johnson Announces Updated Financials and 2023 Guidance Following Completion of the Kenvue\nSeparation\n\nCompany expects increased 2023 Reported Sales Growth of 7.0% - 8.0%, Operational Sales Growth of 7.5% - 8.5%, and\nAdjusted Operational Sales Growth of 6.2% - 7.2%; Figures exclude the COVID-19 Vaccine\n\nCompany expects 2023 Adjusted Reported Earnings Per Share (EPS) of $10.00 - $10.10, reflecting increased growth of\n12.5% at the mid-point and Adjusted Operational EPS of $9.90 - $10.00, reflecting increased growth of 11.5% at the mid-\npoint\n\nCompany reduced outstanding share count by approximately 191 million; 2023 guidance reflects only a partial-year benefit\nof approximately 73.5 million shares or $0.28 benefit to EPS\n\nCompany secured $13.2 billion in cash proceeds from the Kenvue debt offering and initial public offering and maintains 9.5%\nof equity stake in Kenvue\n\nCompany maintains its quarterly dividend of $1.19 per share\nNew Brunswick, N.J. (August 30, 2023) Johnson & Johnson (NYSE: JNJ) (the Company) today announced updates to its financials and\n2023 guidance which reflect its operations as a company focused on transformational innovation in Pharmaceutical and MedTech. The\nCompany has published a recorded webinar for investors to provide additional context behind the updated financials and 2023 guidance\nfound in this release, which may be accessed by visiting the Investors section of the Company's website at webcasts & presentations.\nThe completion of this transaction uniquely positions Johnson & Johnson as a Pharmaceutical and MedTech company focused on delivering\ntransformative healthcare solutions to patients, said Joaquin Duato, Chairman of the Board and Chief Executive Officer. We are incredibly\nproud of the focus and dedication of our employees worldwide to achieve this milestone, which we are confident will unlock near- and long-\nterm value for all of our stakeholders.\nAs previously announced, the Company recently completed an exchange offer to finalize the separation of Kenvue Inc., formerly Johnson &\nJohnsons Consumer Health business. As a result of the completion of the exchange offer, Johnson & Johnson will now present its\nConsumer Health business financial results as discontinued operations, including a gain of approximately $20 billion in the third quarter of\n2023.", "doc_name": "JOHNSON_JOHNSON_2023_8K_dated-2023-08-30", "evidence_page_num": 3, "evidence_text_full_page": "Exhibit 99.1\nJohnson & Johnson Announces Updated Financials and 2023 Guidance Following Completion of the Kenvue\nSeparation\n\nCompany expects increased 2023 Reported Sales Growth of 7.0% - 8.0%, Operational Sales Growth of 7.5% - 8.5%, and\nAdjusted Operational Sales Growth of 6.2% - 7.2%; Figures exclude the COVID-19 Vaccine\n\nCompany expects 2023 Adjusted Reported Earnings Per Share (EPS) of $10.00 - $10.10, reflecting increased growth of\n12.5% at the mid-point and Adjusted Operational EPS of $9.90 - $10.00, reflecting increased growth of 11.5% at the mid-\npoint\n\nCompany reduced outstanding share count by approximately 191 million; 2023 guidance reflects only a partial-year benefit\nof approximately 73.5 million shares or $0.28 benefit to EPS\n\nCompany secured $13.2 billion in cash proceeds from the Kenvue debt offering and initial public offering and maintains 9.5%\nof equity stake in Kenvue\n\nCompany maintains its quarterly dividend of $1.19 per share\nNew Brunswick, N.J. (August 30, 2023) Johnson & Johnson (NYSE: JNJ) (the Company) today announced updates to its financials and\n2023 guidance which reflect its operations as a company focused on transformational innovation in Pharmaceutical and MedTech. The\nCompany has published a recorded webinar for investors to provide additional context behind the updated financials and 2023 guidance\nfound in this release, which may be accessed by visiting the Investors section of the Company's website at webcasts & presentations.\nThe completion of this transaction uniquely positions Johnson & Johnson as a Pharmaceutical and MedTech company focused on delivering\ntransformative healthcare solutions to patients, said Joaquin Duato, Chairman of the Board and Chief Executive Officer. We are incredibly\nproud of the focus and dedication of our employees worldwide to achieve this milestone, which we are confident will unlock near- and long-\nterm value for all of our stakeholders.\nAs previously announced, the Company recently completed an exchange offer to finalize the separation of Kenvue Inc., formerly Johnson &\nJohnsons Consumer Health business. As a result of the completion of the exchange offer, Johnson & Johnson will now present its\nConsumer Health business financial results as discontinued operations, including a gain of approximately $20 billion in the third quarter of\n2023.\n"}]} +{"financebench_id": "financebench_id_01491", "company": "Johnson & Johnson", "doc_name": "JOHNSON_JOHNSON_2023_8K_dated-2023-08-30", "question_type": "novel-generated", "question_reasoning": null, "domain_question_num": null, "question": "What is the amount of the cash proceeds that JnJ realised from the separation of Kenvue (formerly Consumer Health business segment), as of August 30, 2023?", "answer": "JnJ realised $13.2 billion in cash proceeds from the separation of Kenvue.", "justification": null, "dataset_subset_label": "OPEN_SOURCE", "evidence": [{"evidence_text": "Exhibit 99.1\nJohnson & Johnson Announces Updated Financials and 2023 Guidance Following Completion of the Kenvue\nSeparation\n\nCompany expects increased 2023 Reported Sales Growth of 7.0% - 8.0%, Operational Sales Growth of 7.5% - 8.5%, and\nAdjusted Operational Sales Growth of 6.2% - 7.2%; Figures exclude the COVID-19 Vaccine\n\nCompany expects 2023 Adjusted Reported Earnings Per Share (EPS) of $10.00 - $10.10, reflecting increased growth of\n12.5% at the mid-point and Adjusted Operational EPS of $9.90 - $10.00, reflecting increased growth of 11.5% at the mid-\npoint\n\nCompany reduced outstanding share count by approximately 191 million; 2023 guidance reflects only a partial-year benefit\nof approximately 73.5 million shares or $0.28 benefit to EPS\n\nCompany secured $13.2 billion in cash proceeds from the Kenvue debt offering and initial public offering and maintains 9.5%\nof equity stake in Kenvue\n\nCompany maintains its quarterly dividend of $1.19 per share\nNew Brunswick, N.J. (August 30, 2023) Johnson & Johnson (NYSE: JNJ) (the Company) today announced updates to its financials and\n2023 guidance which reflect its operations as a company focused on transformational innovation in Pharmaceutical and MedTech. The\nCompany has published a recorded webinar for investors to provide additional context behind the updated financials and 2023 guidance\nfound in this release, which may be accessed by visiting the Investors section of the Company's website at webcasts & presentations.\nThe completion of this transaction uniquely positions Johnson & Johnson as a Pharmaceutical and MedTech company focused on delivering\ntransformative healthcare solutions to patients, said Joaquin Duato, Chairman of the Board and Chief Executive Officer. We are incredibly\nproud of the focus and dedication of our employees worldwide to achieve this milestone, which we are confident will unlock near- and long-\nterm value for all of our stakeholders.\nAs previously announced, the Company recently completed an exchange offer to finalize the separation of Kenvue Inc., formerly Johnson &\nJohnsons Consumer Health business. As a result of the completion of the exchange offer, Johnson & Johnson will now present its\nConsumer Health business financial results as discontinued operations, including a gain of approximately $20 billion in the third quarter of\n2023.", "doc_name": "JOHNSON_JOHNSON_2023_8K_dated-2023-08-30", "evidence_page_num": 3, "evidence_text_full_page": "Exhibit 99.1\nJohnson & Johnson Announces Updated Financials and 2023 Guidance Following Completion of the Kenvue\nSeparation\n\nCompany expects increased 2023 Reported Sales Growth of 7.0% - 8.0%, Operational Sales Growth of 7.5% - 8.5%, and\nAdjusted Operational Sales Growth of 6.2% - 7.2%; Figures exclude the COVID-19 Vaccine\n\nCompany expects 2023 Adjusted Reported Earnings Per Share (EPS) of $10.00 - $10.10, reflecting increased growth of\n12.5% at the mid-point and Adjusted Operational EPS of $9.90 - $10.00, reflecting increased growth of 11.5% at the mid-\npoint\n\nCompany reduced outstanding share count by approximately 191 million; 2023 guidance reflects only a partial-year benefit\nof approximately 73.5 million shares or $0.28 benefit to EPS\n\nCompany secured $13.2 billion in cash proceeds from the Kenvue debt offering and initial public offering and maintains 9.5%\nof equity stake in Kenvue\n\nCompany maintains its quarterly dividend of $1.19 per share\nNew Brunswick, N.J. (August 30, 2023) Johnson & Johnson (NYSE: JNJ) (the Company) today announced updates to its financials and\n2023 guidance which reflect its operations as a company focused on transformational innovation in Pharmaceutical and MedTech. The\nCompany has published a recorded webinar for investors to provide additional context behind the updated financials and 2023 guidance\nfound in this release, which may be accessed by visiting the Investors section of the Company's website at webcasts & presentations.\nThe completion of this transaction uniquely positions Johnson & Johnson as a Pharmaceutical and MedTech company focused on delivering\ntransformative healthcare solutions to patients, said Joaquin Duato, Chairman of the Board and Chief Executive Officer. We are incredibly\nproud of the focus and dedication of our employees worldwide to achieve this milestone, which we are confident will unlock near- and long-\nterm value for all of our stakeholders.\nAs previously announced, the Company recently completed an exchange offer to finalize the separation of Kenvue Inc., formerly Johnson &\nJohnsons Consumer Health business. As a result of the completion of the exchange offer, Johnson & Johnson will now present its\nConsumer Health business financial results as discontinued operations, including a gain of approximately $20 billion in the third quarter of\n2023.\n"}]} +{"financebench_id": "financebench_id_00601", "company": "Ulta Beauty", "doc_name": "ULTABEAUTY_2023Q4_EARNINGS", "question_type": "novel-generated", "question_reasoning": null, "domain_question_num": null, "question": "What drove the reduction in SG&A expense as a percent of net sales in FY2023?", "answer": "Lower marketing expenses and leverage of incentive compensation due to higher sales. The answer here assumes FY2023 refers to the 12 months ended on January 28, 2023 (although the company refers to this period as its fiscal 2022.", "justification": "Fiscal 2022 = FY2023. Fiscal 2021 = FY2022.", "dataset_subset_label": "OPEN_SOURCE", "evidence": [{"evidence_text": "For the Full Year of Fiscal 2022\nNet sales increased 18.3% to $10.2 billion compared to $8.6 billion in fiscal 2021,\nprimarily due to the favorable impact from the continued resilience of the beauty\ncategory, retail price increases, the impact of new brands and product innovation,\nincreased social occasions, and fewer COVID-19 limitations compared to fiscal 2021.\nComparable sales increased 15.6% compared to an increase of 37.9% in fiscal 2021,\ndriven by a 10.8% increase in transactions and a 4.3% increase in average ticket.\nGross profit increased 20.1% to $4.0 billion compared to $3.4 billion in fiscal 2021. As\na percentage of net sales, gross profit increased to 39.6% compared to 39.0% in fiscal\n2021, primarily due to leverage of fixed costs, strong growth in other revenue, and\nfavorable channel mix shifts, partially offset by higher inventory shrink and lower\nmerchandise margin.\nSG&A expenses increased 16.2% to $2.4 billion compared to $2.1 billion in fiscal\n2021. As a percentage of net sales, SG&A expenses decreased to 23.5% compared to\n23.9% in fiscal 2021, primarily due to lower marketing expenses and leverage of\nincentive compensation due to higher sales, partially offset by deleverage of corporate\noverhead due to strategic investments and deleverage of store payroll and benefits\ndue to wage investments.", "doc_name": "ULTABEAUTY_2023Q4_EARNINGS", "evidence_page_num": 1, "evidence_text_full_page": "Comparable sales (sales for stores open at least 14 months and e-commerce sales)\nincreased 15.6% compared to an increase of 21.4% in the fourth quarter of fiscal 2021,\ndriven by a 13.6% increase in transactions and a 1.8% increase in average ticket.\nGross profit increased 18.0% to $1.2 billion compared to $1.0 billion in the fourth\nquarter of fiscal 2021. As a percentage of net sales, gross profit of 37.6% was flat\ncompared to the fourth quarter of fiscal 2021, primarily due to leverage of fixed costs,\nfavorable channel mix shifts, and strong growth in other revenue, offset by higher\ninventory shrink.\nSelling, general and administrative (SG&A) expenses increased 17.3% to $762.7\nmillion compared to $650.0 million in the fourth quarter of fiscal 2021. As a percentage\nof net sales, SG&A expenses decreased to 23.6% compared to 23.8% in the fourth\nquarter of fiscal 2021, primarily due to leverage of marketing expenses and incentive\ncompensation due to higher sales, partially offset by deleverage of store payroll and\nbenefits due to wage investments and deleverage in corporate overhead due to\nstrategic investments.\nOperating income increased 19.2% to $447.6 million, or 13.9% of net sales, compared\nto $375.6 million, or 13.8% of net sales, in the fourth quarter of fiscal 2021.\nThe tax rate increased to 24.6% compared to 22.9% in the fourth quarter of fiscal\n2021.\nNet income increased 17.8% to $340.8 million compared to $289.4 million in the fourth\nquarter of fiscal 2021.\nDiluted earnings per share increased 23.5% to $6.68, including a $0.02 benefit due to\nincome tax accounting for stock-based compensation, compared to $5.41 including a\n$0.05 benefit due to income tax accounting for stock-based compensation, in the fourth\nquarter of fiscal 2021.\nFor the Full Year of Fiscal 2022\nNet sales increased 18.3% to $10.2 billion compared to $8.6 billion in fiscal 2021,\nprimarily due to the favorable impact from the continued resilience of the beauty\ncategory, retail price increases, the impact of new brands and product innovation,\nincreased social occasions, and fewer COVID-19 limitations compared to fiscal 2021.\nComparable sales increased 15.6% compared to an increase of 37.9% in fiscal 2021,\ndriven by a 10.8% increase in transactions and a 4.3% increase in average ticket.\nGross profit increased 20.1% to $4.0 billion compared to $3.4 billion in fiscal 2021. As\na percentage of net sales, gross profit increased to 39.6% compared to 39.0% in fiscal\n2021, primarily due to leverage of fixed costs, strong growth in other revenue, and\nfavorable channel mix shifts, partially offset by higher inventory shrink and lower\nmerchandise margin.\nSG&A expenses increased 16.2% to $2.4 billion compared to $2.1 billion in fiscal\n2021. As a percentage of net sales, SG&A expenses decreased to 23.5% compared to\n23.9% in fiscal 2021, primarily due to lower marketing expenses and leverage of\nincentive compensation due to higher sales, partially offset by deleverage of corporate\noverhead due to strategic investments and deleverage of store payroll and benefits\ndue to wage investments.\nOperating income increased 26.3% to $1.6 billion, or 16.1% of net sales, compared to\n$1.3 billion, or 15.0% of net sales, in fiscal 2021.\nThe tax rate increased to 24.4% compared to 23.9% in fiscal 2021.\nNet income increased 26.0% to $1.2 billion compared to $985.8 million in fiscal 2021.\n"}]} +{"financebench_id": "financebench_id_00603", "company": "Ulta Beauty", "doc_name": "ULTABEAUTY_2023Q4_EARNINGS", "question_type": "novel-generated", "question_reasoning": null, "domain_question_num": null, "question": "What drove the increase in Ulta Beauty's merchandise inventories balance at end of FY2023?", "answer": "Increase in Merchandise inventories balance was driven by the opening of 47 new stores. The answer here assumes FY2023 refers to the 12 months ended on January 28, 2023 (although the company refers to this period as its fiscal 2022.", "justification": "Fiscal 2022 = FY2023. Fiscal 2021 = FY2022.", "dataset_subset_label": "OPEN_SOURCE", "evidence": [{"evidence_text": "Balance Sheet\nCash and cash equivalents at the end of the fourth quarter of fiscal 2022 were $737.9\nmillion.\nMerchandise inventories, net at the end of the fourth quarter of fiscal 2022 totaled $1.6\nbillion compared to $1.5 billion at the end of the fourth quarter of fiscal 2021. The $104.2\nmillion increase was primarily due to the opening of 47 new stores since January 29, 2022,\ninventory to support new brand launches and brand expansions, and inventory cost\nincreases.", "doc_name": "ULTABEAUTY_2023Q4_EARNINGS", "evidence_page_num": 2, "evidence_text_full_page": "Diluted earnings per share increased 33.5% to $24.01, including a $0.07 benefit due to\nincome tax accounting for stock-based compensation, compared to $17.98 including a\n$0.13 benefit due to income tax accounting for stock-based compensation, in fiscal\n2021.\nBalance Sheet\nCash and cash equivalents at the end of the fourth quarter of fiscal 2022 were $737.9\nmillion.\nMerchandise inventories, net at the end of the fourth quarter of fiscal 2022 totaled $1.6\nbillion compared to $1.5 billion at the end of the fourth quarter of fiscal 2021. The $104.2\nmillion increase was primarily due to the opening of 47 new stores since January 29, 2022,\ninventory to support new brand launches and brand expansions, and inventory cost\nincreases.\nShare Repurchase Program\nDuring the fourth quarter of fiscal 2022, the Company repurchased 722,457 shares of its\ncommon stock at a cost of $328.1 million. During fiscal 2022, the Company repurchased 2.2\nmillion shares of its common stock at a cost of $900.0 million. As of January 28, 2023, $1.1\nbillion remained available under the $2.0 billion share repurchase program announced in\nMarch 2022.\nStore Update\nReal estate activity in the fourth quarter of fiscal 2022 included 12 new stores located in\nGarden Grove, CA; Glendale, AZ; Hartsdale, NY; Hollister, CA; Indianapolis, IN; Liverpool,\nNY; Nanuet, NY; Oklahoma City, OK; Richmond, TX; Rock Springs, WY; Tullahoma, TN;\nand Woburn, MA. In addition, the Company relocated one store and remodeled 12 stores.\nDuring fiscal 2022, the Company opened 47 new stores, relocated 12 stores, and remodeled\n20 stores.\nAt the end of the fourth quarter of fiscal 2022, the Company operated 1,355 stores totaling\n14.2 million square feet.\nFiscal 2023 Outlook\nFor fiscal 2023, the Company plans to:\n \n \n \n \n \nFY23 Outlook\nNet sales\n \n \n$10.95 billion to $11.05 billion\nComparable sales\n \n \n4% to 5%\nNew stores, net\n \n \n25-30\nRemodel and relocation projects\n \n \n20-30\nOperating margin\n \n \n14.7% to 15.0%\nDiluted earnings per share\n \n \n$24.70 to $25.40\nShare repurchases\n \n \napproximately $900 million\nEffective tax rate\n \n \napproximately 24.6%\nCapital expenditures\n \n \n$400 million to $475 million\nDepreciation and amortization expense\n \n \n$245 million to $250 million\n"}]} +{"financebench_id": "financebench_id_00605", "company": "Ulta Beauty", "doc_name": "ULTABEAUTY_2023Q4_EARNINGS", "question_type": "novel-generated", "question_reasoning": null, "domain_question_num": null, "question": "What percent of Ulta Beauty's total spend on stock repurchases for FY 2023 occurred in Q4 of FY2023?", "answer": "36%. The answer here assumes FY2023 refers to the 12 months ended on January 28, 2023 (although the company refers to this period as its fiscal 2022.", "justification": "Fiscal 2022 = FY2023. Fiscal 2021 = FY2022. Percent spent in Q4 of FY2023 = Amount spent in Q4 of FY2023/Total amount spent in FY2023*100 =$328.1 million /$900 million * 100 = 36%", "dataset_subset_label": "OPEN_SOURCE", "evidence": [{"evidence_text": "Share Repurchase Program\nDuring the fourth quarter of fiscal 2022, the Company repurchased 722,457 shares of its\ncommon stock at a cost of $328.1 million. During fiscal 2022, the Company repurchased 2.2\nmillion shares of its common stock at a cost of $900.0 million. As of January 28, 2023, $1.1\nbillion remained available under the $2.0 billion share repurchase program announced in\nMarch 2022.", "doc_name": "ULTABEAUTY_2023Q4_EARNINGS", "evidence_page_num": 2, "evidence_text_full_page": "Diluted earnings per share increased 33.5% to $24.01, including a $0.07 benefit due to\nincome tax accounting for stock-based compensation, compared to $17.98 including a\n$0.13 benefit due to income tax accounting for stock-based compensation, in fiscal\n2021.\nBalance Sheet\nCash and cash equivalents at the end of the fourth quarter of fiscal 2022 were $737.9\nmillion.\nMerchandise inventories, net at the end of the fourth quarter of fiscal 2022 totaled $1.6\nbillion compared to $1.5 billion at the end of the fourth quarter of fiscal 2021. The $104.2\nmillion increase was primarily due to the opening of 47 new stores since January 29, 2022,\ninventory to support new brand launches and brand expansions, and inventory cost\nincreases.\nShare Repurchase Program\nDuring the fourth quarter of fiscal 2022, the Company repurchased 722,457 shares of its\ncommon stock at a cost of $328.1 million. During fiscal 2022, the Company repurchased 2.2\nmillion shares of its common stock at a cost of $900.0 million. As of January 28, 2023, $1.1\nbillion remained available under the $2.0 billion share repurchase program announced in\nMarch 2022.\nStore Update\nReal estate activity in the fourth quarter of fiscal 2022 included 12 new stores located in\nGarden Grove, CA; Glendale, AZ; Hartsdale, NY; Hollister, CA; Indianapolis, IN; Liverpool,\nNY; Nanuet, NY; Oklahoma City, OK; Richmond, TX; Rock Springs, WY; Tullahoma, TN;\nand Woburn, MA. In addition, the Company relocated one store and remodeled 12 stores.\nDuring fiscal 2022, the Company opened 47 new stores, relocated 12 stores, and remodeled\n20 stores.\nAt the end of the fourth quarter of fiscal 2022, the Company operated 1,355 stores totaling\n14.2 million square feet.\nFiscal 2023 Outlook\nFor fiscal 2023, the Company plans to:\n \n \n \n \n \nFY23 Outlook\nNet sales\n \n \n$10.95 billion to $11.05 billion\nComparable sales\n \n \n4% to 5%\nNew stores, net\n \n \n25-30\nRemodel and relocation projects\n \n \n20-30\nOperating margin\n \n \n14.7% to 15.0%\nDiluted earnings per share\n \n \n$24.70 to $25.40\nShare repurchases\n \n \napproximately $900 million\nEffective tax rate\n \n \napproximately 24.6%\nCapital expenditures\n \n \n$400 million to $475 million\nDepreciation and amortization expense\n \n \n$245 million to $250 million\n"}]} +{"financebench_id": "financebench_id_00606", "company": "Ulta Beauty", "doc_name": "ULTABEAUTY_2023Q4_EARNINGS", "question_type": "novel-generated", "question_reasoning": null, "domain_question_num": null, "question": "Did Ulta Beauty's wages expense as a percent of net sales increase or decrease in FY2023?", "answer": "Wages expense as a percent of net sales increased in FY2023. The answer here assumes FY2023 refers to the 12 months ended on January 28, 2023 (although the company refers to this period as its fiscal 2022.", "justification": "Fiscal 2022 = FY2023. Fiscal 2021 = FY2022. Store payroll and benefits = wages. Store payroll and benefits offsets reduction in SG&A percent of net sales in FY2023.", "dataset_subset_label": "OPEN_SOURCE", "evidence": [{"evidence_text": "For the Full Year of Fiscal 2022\nNet sales increased 18.3% to $10.2 billion compared to $8.6 billion in fiscal 2021,\nprimarily due to the favorable impact from the continued resilience of the beauty\ncategory, retail price increases, the impact of new brands and product innovation,\nincreased social occasions, and fewer COVID-19 limitations compared to fiscal 2021.\nComparable sales increased 15.6% compared to an increase of 37.9% in fiscal 2021,\ndriven by a 10.8% increase in transactions and a 4.3% increase in average ticket.\nGross profit increased 20.1% to $4.0 billion compared to $3.4 billion in fiscal 2021. As\na percentage of net sales, gross profit increased to 39.6% compared to 39.0% in fiscal\n2021, primarily due to leverage of fixed costs, strong growth in other revenue, and\nfavorable channel mix shifts, partially offset by higher inventory shrink and lower\nmerchandise margin.\nSG&A expenses increased 16.2% to $2.4 billion compared to $2.1 billion in fiscal\n2021. As a percentage of net sales, SG&A expenses decreased to 23.5% compared to\n23.9% in fiscal 2021, primarily due to lower marketing expenses and leverage of\nincentive compensation due to higher sales, partially offset by deleverage of corporate\noverhead due to strategic investments and deleverage of store payroll and benefits\ndue to wage investments.", "doc_name": "ULTABEAUTY_2023Q4_EARNINGS", "evidence_page_num": 1, "evidence_text_full_page": "Comparable sales (sales for stores open at least 14 months and e-commerce sales)\nincreased 15.6% compared to an increase of 21.4% in the fourth quarter of fiscal 2021,\ndriven by a 13.6% increase in transactions and a 1.8% increase in average ticket.\nGross profit increased 18.0% to $1.2 billion compared to $1.0 billion in the fourth\nquarter of fiscal 2021. As a percentage of net sales, gross profit of 37.6% was flat\ncompared to the fourth quarter of fiscal 2021, primarily due to leverage of fixed costs,\nfavorable channel mix shifts, and strong growth in other revenue, offset by higher\ninventory shrink.\nSelling, general and administrative (SG&A) expenses increased 17.3% to $762.7\nmillion compared to $650.0 million in the fourth quarter of fiscal 2021. As a percentage\nof net sales, SG&A expenses decreased to 23.6% compared to 23.8% in the fourth\nquarter of fiscal 2021, primarily due to leverage of marketing expenses and incentive\ncompensation due to higher sales, partially offset by deleverage of store payroll and\nbenefits due to wage investments and deleverage in corporate overhead due to\nstrategic investments.\nOperating income increased 19.2% to $447.6 million, or 13.9% of net sales, compared\nto $375.6 million, or 13.8% of net sales, in the fourth quarter of fiscal 2021.\nThe tax rate increased to 24.6% compared to 22.9% in the fourth quarter of fiscal\n2021.\nNet income increased 17.8% to $340.8 million compared to $289.4 million in the fourth\nquarter of fiscal 2021.\nDiluted earnings per share increased 23.5% to $6.68, including a $0.02 benefit due to\nincome tax accounting for stock-based compensation, compared to $5.41 including a\n$0.05 benefit due to income tax accounting for stock-based compensation, in the fourth\nquarter of fiscal 2021.\nFor the Full Year of Fiscal 2022\nNet sales increased 18.3% to $10.2 billion compared to $8.6 billion in fiscal 2021,\nprimarily due to the favorable impact from the continued resilience of the beauty\ncategory, retail price increases, the impact of new brands and product innovation,\nincreased social occasions, and fewer COVID-19 limitations compared to fiscal 2021.\nComparable sales increased 15.6% compared to an increase of 37.9% in fiscal 2021,\ndriven by a 10.8% increase in transactions and a 4.3% increase in average ticket.\nGross profit increased 20.1% to $4.0 billion compared to $3.4 billion in fiscal 2021. As\na percentage of net sales, gross profit increased to 39.6% compared to 39.0% in fiscal\n2021, primarily due to leverage of fixed costs, strong growth in other revenue, and\nfavorable channel mix shifts, partially offset by higher inventory shrink and lower\nmerchandise margin.\nSG&A expenses increased 16.2% to $2.4 billion compared to $2.1 billion in fiscal\n2021. As a percentage of net sales, SG&A expenses decreased to 23.5% compared to\n23.9% in fiscal 2021, primarily due to lower marketing expenses and leverage of\nincentive compensation due to higher sales, partially offset by deleverage of corporate\noverhead due to strategic investments and deleverage of store payroll and benefits\ndue to wage investments.\nOperating income increased 26.3% to $1.6 billion, or 16.1% of net sales, compared to\n$1.3 billion, or 15.0% of net sales, in fiscal 2021.\nThe tax rate increased to 24.4% compared to 23.9% in fiscal 2021.\nNet income increased 26.0% to $1.2 billion compared to $985.8 million in fiscal 2021.\n"}]} diff --git a/benchmark/RAG/datasets/Benchmark_Lite/Financebench/pdfs/AES_2022_10K.pdf b/benchmark/RAG/datasets/Benchmark_Lite/Financebench/pdfs/AES_2022_10K.pdf new file mode 100644 index 00000000..65ea6bb7 Binary files /dev/null and b/benchmark/RAG/datasets/Benchmark_Lite/Financebench/pdfs/AES_2022_10K.pdf differ diff --git a/benchmark/RAG/datasets/Benchmark_Lite/Financebench/pdfs/BESTBUY_2023_10K.pdf b/benchmark/RAG/datasets/Benchmark_Lite/Financebench/pdfs/BESTBUY_2023_10K.pdf new file mode 100644 index 00000000..bf024d86 Binary files /dev/null and b/benchmark/RAG/datasets/Benchmark_Lite/Financebench/pdfs/BESTBUY_2023_10K.pdf differ diff --git a/benchmark/RAG/datasets/Benchmark_Lite/Financebench/pdfs/JOHNSON_JOHNSON_2023_8K_dated-2023-08-30.pdf b/benchmark/RAG/datasets/Benchmark_Lite/Financebench/pdfs/JOHNSON_JOHNSON_2023_8K_dated-2023-08-30.pdf new file mode 100644 index 00000000..f4fa338d Binary files /dev/null and b/benchmark/RAG/datasets/Benchmark_Lite/Financebench/pdfs/JOHNSON_JOHNSON_2023_8K_dated-2023-08-30.pdf differ diff --git a/benchmark/RAG/datasets/Benchmark_Lite/Financebench/pdfs/ULTABEAUTY_2023Q4_EARNINGS.pdf b/benchmark/RAG/datasets/Benchmark_Lite/Financebench/pdfs/ULTABEAUTY_2023Q4_EARNINGS.pdf new file mode 100644 index 00000000..d88813e1 Binary files /dev/null and b/benchmark/RAG/datasets/Benchmark_Lite/Financebench/pdfs/ULTABEAUTY_2023Q4_EARNINGS.pdf differ diff --git a/benchmark/RAG/datasets/Benchmark_Lite/Locomo/Locomo.json b/benchmark/RAG/datasets/Benchmark_Lite/Locomo/Locomo.json new file mode 100644 index 00000000..3a15de89 --- /dev/null +++ b/benchmark/RAG/datasets/Benchmark_Lite/Locomo/Locomo.json @@ -0,0 +1,11364 @@ +[ + { + "sample_id": "conv-26", + "conversation": { + "speaker_a": "Caroline", + "speaker_b": "Melanie", + "session_1_date_time": "1:56 pm on 8 May, 2023", + "session_1": [ + { + "speaker": "Caroline", + "dia_id": "D1:1", + "text": "Hey Mel! Good to see you! How have you been?" + }, + { + "speaker": "Melanie", + "dia_id": "D1:2", + "text": "Hey Caroline! Good to see you! I'm swamped with the kids & work. What's up with you? Anything new?" + }, + { + "speaker": "Caroline", + "dia_id": "D1:3", + "text": "I went to a LGBTQ support group yesterday and it was so powerful." + }, + { + "speaker": "Melanie", + "dia_id": "D1:4", + "text": "Wow, that's cool, Caroline! What happened that was so awesome? Did you hear any inspiring stories?" + }, + { + "speaker": "Caroline", + "img_url": [ + "https://i.redd.it/l7hozpetnhlb1.jpg" + ], + "blip_caption": "a photo of a dog walking past a wall with a painting of a woman", + "query": "transgender pride flag mural", + "dia_id": "D1:5", + "text": "The transgender stories were so inspiring! I was so happy and thankful for all the support." + }, + { + "speaker": "Melanie", + "dia_id": "D1:6", + "text": "Wow, love that painting! So cool you found such a helpful group. What's it done for you?" + }, + { + "speaker": "Caroline", + "dia_id": "D1:7", + "text": "The support group has made me feel accepted and given me courage to embrace myself." + }, + { + "speaker": "Melanie", + "dia_id": "D1:8", + "text": "That's really cool. You've got guts. What now?" + }, + { + "speaker": "Caroline", + "dia_id": "D1:9", + "text": "Gonna continue my edu and check out career options, which is pretty exciting!" + }, + { + "speaker": "Melanie", + "dia_id": "D1:10", + "text": "Wow, Caroline! What kinda jobs are you thinkin' of? Anything that stands out?" + }, + { + "speaker": "Caroline", + "dia_id": "D1:11", + "text": "I'm keen on counseling or working in mental health - I'd love to support those with similar issues." + }, + { + "speaker": "Melanie", + "img_url": [ + "http://candicealexander.com/cdn/shop/products/IMG_7269_a49d5af8-c76c-4ecd-ae20-48c08cb11dec.jpg" + ], + "blip_caption": "a photo of a painting of a sunset over a lake", + "query": "painting sunrise", + "dia_id": "D1:12", + "text": "You'd be a great counselor! Your empathy and understanding will really help the people you work with. By the way, take a look at this." + }, + { + "speaker": "Caroline", + "dia_id": "D1:13", + "text": "Thanks, Melanie! That's really sweet. Is this your own painting?" + }, + { + "speaker": "Melanie", + "dia_id": "D1:14", + "text": "Yeah, I painted that lake sunrise last year! It's special to me." + }, + { + "speaker": "Caroline", + "dia_id": "D1:15", + "text": "Wow, Melanie! The colors really blend nicely. Painting looks like a great outlet for expressing yourself." + }, + { + "speaker": "Melanie", + "dia_id": "D1:16", + "text": "Thanks, Caroline! Painting's a fun way to express my feelings and get creative. It's a great way to relax after a long day." + }, + { + "speaker": "Caroline", + "dia_id": "D1:17", + "text": "Totally agree, Mel. Relaxing and expressing ourselves is key. Well, I'm off to go do some research." + }, + { + "speaker": "Melanie", + "dia_id": "D1:18", + "text": "Yep, Caroline. Taking care of ourselves is vital. I'm off to go swimming with the kids. Talk to you soon!" + } + ], + "session_2_date_time": "1:14 pm on 25 May, 2023", + "session_2": [ + { + "speaker": "Melanie", + "dia_id": "D2:1", + "text": "Hey Caroline, since we last chatted, I've had a lot of things happening to me. I ran a charity race for mental health last Saturday – it was really rewarding. Really made me think about taking care of our minds." + }, + { + "speaker": "Caroline", + "dia_id": "D2:2", + "text": "That charity race sounds great, Mel! Making a difference & raising awareness for mental health is super rewarding - I'm really proud of you for taking part!" + }, + { + "speaker": "Melanie", + "dia_id": "D2:3", + "text": "Thanks, Caroline! The event was really thought-provoking. I'm starting to realize that self-care is really important. It's a journey for me, but when I look after myself, I'm able to better look after my family." + }, + { + "speaker": "Caroline", + "dia_id": "D2:4", + "text": "I totally agree, Melanie. Taking care of ourselves is so important - even if it's not always easy. Great that you're prioritizing self-care." + }, + { + "speaker": "Melanie", + "dia_id": "D2:5", + "text": "Yeah, it's tough. So I'm carving out some me-time each day - running, reading, or playing my violin - which refreshes me and helps me stay present for my fam!" + }, + { + "speaker": "Caroline", + "dia_id": "D2:6", + "text": "That's great, Mel! Taking time for yourself is so important. You're doing an awesome job looking after yourself and your family!" + }, + { + "speaker": "Melanie", + "dia_id": "D2:7", + "text": "Thanks, Caroline. It's still a work in progress, but I'm doing my best. My kids are so excited about summer break! We're thinking about going camping next month. Any fun plans for the summer?" + }, + { + "speaker": "Caroline", + "dia_id": "D2:8", + "text": "Researching adoption agencies — it's been a dream to have a family and give a loving home to kids who need it." + }, + { + "speaker": "Melanie", + "dia_id": "D2:9", + "text": "Wow, Caroline! That's awesome! Taking in kids in need - you're so kind. Your future family is gonna be so lucky to have you!" + }, + { + "speaker": "Caroline", + "img_url": [ + "https://live.staticflickr.com/3437/3935231341_b2955b00dd_b.jpg" + ], + "blip_caption": "a photography of a sign for a new arrival and an information and domestic building", + "query": "adoption agency brochure", + "dia_id": "D2:10", + "re-download": true, + "text": "Thanks, Mel! My goal is to give kids a loving home. I'm truly grateful for all the support I've got from friends and mentors. Now the hard work starts to turn my dream into a reality. And here's one of the adoption agencies I'm looking into. It's a lot to take in, but I'm feeling hopeful and optimistic." + }, + { + "speaker": "Melanie", + "dia_id": "D2:11", + "text": "Wow, that agency looks great! What made you pick it?" + }, + { + "speaker": "Caroline", + "dia_id": "D2:12", + "text": "I chose them 'cause they help LGBTQ+ folks with adoption. Their inclusivity and support really spoke to me." + }, + { + "speaker": "Melanie", + "dia_id": "D2:13", + "text": "That's great, Caroline! Loving the inclusivity and support. Anything you're excited for in the adoption process?" + }, + { + "speaker": "Caroline", + "dia_id": "D2:14", + "text": "I'm thrilled to make a family for kids who need one. It'll be tough as a single parent, but I'm up for the challenge!" + }, + { + "speaker": "Melanie", + "dia_id": "D2:15", + "text": "You're doing something amazing! Creating a family for those kids is so lovely. You'll be an awesome mom! Good luck!" + }, + { + "speaker": "Caroline", + "dia_id": "D2:16", + "text": "Thanks, Melanie! Your kind words really mean a lot. I'll do my best to make sure these kids have a safe and loving home." + }, + { + "speaker": "Melanie", + "dia_id": "D2:17", + "text": "No doubts, Caroline. You have such a caring heart - they'll get all the love and stability they need! Excited for this new chapter!" + } + ], + "session_3_date_time": "7:55 pm on 9 June, 2023", + "session_3": [ + { + "speaker": "Caroline", + "dia_id": "D3:1", + "text": "Hey Melanie! How's it going? I wanted to tell you about my school event last week. It was awesome! I talked about my transgender journey and encouraged students to get involved in the LGBTQ community. It was great to see their reactions. It made me reflect on how far I've come since I started transitioning three years ago." + }, + { + "speaker": "Melanie", + "dia_id": "D3:2", + "text": "Hey Caroline! Great to hear from you. Sounds like your event was amazing! I'm so proud of you for spreading awareness and getting others involved in the LGBTQ community. You've come a long way since your transition - keep on inspiring people with your strength and courage!" + }, + { + "speaker": "Caroline", + "dia_id": "D3:3", + "text": "Thanks, Mel! Your backing really means a lot. I felt super powerful giving my talk. I shared my own journey, the struggles I had and how much I've developed since coming out. It was wonderful to see how the audience related to what I said and how it inspired them to be better allies. Conversations about gender identity and inclusion are so necessary and I'm thankful for being able to give a voice to the trans community." + }, + { + "speaker": "Melanie", + "dia_id": "D3:4", + "text": "Wow, Caroline, you're doing an awesome job of inspiring others with your journey. It's great to be part of it and see how you're positively affecting so many. Talking about inclusivity and acceptance is crucial, and you're so brave to speak up for the trans community. Keep up the great work!" + }, + { + "speaker": "Caroline", + "dia_id": "D3:5", + "text": "Thanks Mel! Your kind words mean a lot. Sharing our experiences isn't always easy, but I feel it's important to help promote understanding and acceptance. I've been blessed with loads of love and support throughout this journey, and I want to pass it on to others. By sharing our stories, we can build a strong, supportive community of hope." + }, + { + "speaker": "Melanie", + "dia_id": "D3:6", + "text": "Yeah, Caroline! It takes courage to talk about our own stories. But it's in these vulnerable moments that we bond and understand each other. We all have our different paths, but if we share them, we show people that they're not alone. Our stories can be so inspiring and encouraging to others who are facing the same challenges. Thank you for using your voice to create love, acceptance, and hope. You're doing amazing!" + }, + { + "speaker": "Caroline", + "dia_id": "D3:7", + "text": "Your words mean a lot to me. I'm grateful for the chance to share my story and give others hope. We all have unique paths, and by working together we can build a more inclusive and understanding world. I'm going to keep using my voice to make a change and lift others up. And you're part of that!" + }, + { + "speaker": "Melanie", + "dia_id": "D3:8", + "text": "Thanks, Caroline, for letting me join your journey. I'm so proud to be part of the difference you're making. Let's keep motivating and helping each other out as we journey through life. We can make a real impact together!" + }, + { + "speaker": "Caroline", + "dia_id": "D3:9", + "text": "Yeah Mel, let's spread love and understanding! Thanks for the support and encouragement. We can tackle life's challenges together! We got this!" + }, + { + "speaker": "Melanie", + "dia_id": "D3:10", + "text": "Yes, Caroline! We can do it. Your courage is inspiring. I want to be couragous for my family- they motivate me and give me love. What motivates you?" + }, + { + "speaker": "Caroline", + "img_url": [ + "https://fox2now.com/wp-content/uploads/sites/14/2023/08/that-tall-family.jpg" + ], + "blip_caption": "a photo of a family posing for a picture in a yard", + "query": "group of friends and family", + "dia_id": "D3:11", + "text": "Thanks, Mel! My friends, family and mentors are my rocks – they motivate me and give me the strength to push on. Here's a pic from when we met up last week!" + }, + { + "speaker": "Melanie", + "dia_id": "D3:12", + "text": "Wow, that photo is great! How long have you had such a great support system?" + }, + { + "speaker": "Caroline", + "dia_id": "D3:13", + "text": "Yeah, I'm really lucky to have them. They've been there through everything, I've known these friends for 4 years, since I moved from my home country. Their love and help have been so important especially after that tough breakup. I'm super thankful. Who supports you, Mel?" + }, + { + "speaker": "Melanie", + "img_url": [ + "https://mrswebersneighborhood.com/wp-content/uploads/2022/07/Cedar-Falls-Hocking-Hills.jpg" + ], + "blip_caption": "a photo of a man and a little girl standing in front of a waterfall", + "query": "husband kids hiking nature", + "dia_id": "D3:14", + "text": "I'm lucky to have my husband and kids; they keep me motivated." + }, + { + "speaker": "Caroline", + "dia_id": "D3:15", + "text": "Wow, what an amazing family pic! How long have you been married?" + }, + { + "speaker": "Melanie", + "img_url": [ + "https://i.redd.it/8o28nfllf3eb1.jpg" + ], + "blip_caption": "a photo of a bride in a wedding dress holding a bouquet", + "query": "wedding day", + "dia_id": "D3:16", + "text": "5 years already! Time flies- feels like just yesterday I put this dress on! Thanks, Caroline!" + }, + { + "speaker": "Caroline", + "dia_id": "D3:17", + "text": "Congrats, Melanie! You both looked so great on your wedding day! Wishing you many happy years together!" + }, + { + "speaker": "Melanie", + "img_url": [ + "http://shirleyswardrobe.com/wp-content/uploads/2017/07/LF-Picnic-6.jpg" + ], + "blip_caption": "a photo of a man and woman sitting on a blanket eating food", + "query": "family picnic park laughing", + "dia_id": "D3:18", + "text": "Thanks, Caroline! Appreciate your kind words. Looking forward to more happy years. Our family and moments make it all worth it." + }, + { + "speaker": "Caroline", + "dia_id": "D3:19", + "text": "Looks like you had a great day! How was it? You all look so happy!" + }, + { + "speaker": "Melanie", + "dia_id": "D3:20", + "text": "It so fun! We played games, ate good food, and just hung out together. Family moments make life awesome." + }, + { + "speaker": "Caroline", + "dia_id": "D3:21", + "text": "Sounds great, Mel! Glad you had a great time. Cherish the moments - they're the best!" + }, + { + "speaker": "Melanie", + "dia_id": "D3:22", + "text": "Absolutely, Caroline! I cherish time with family. It's when I really feel alive and happy." + }, + { + "speaker": "Caroline", + "dia_id": "D3:23", + "text": "I 100% agree, Mel. Hanging with loved ones is amazing and brings so much happiness. Those moments really make me thankful. Family is everything." + } + ], + "session_4_date_time": "10:37 am on 27 June, 2023", + "session_4": [ + { + "speaker": "Caroline", + "img_url": [ + "https://i.redd.it/67uas3gnmz7b1.jpg" + ], + "blip_caption": "a photo of a person holding a necklace with a cross and a heart", + "query": "pendant transgender symbol", + "dia_id": "D4:1", + "text": "Hey Melanie! Long time no talk! A lot's been going on in my life! Take a look at this." + }, + { + "speaker": "Melanie", + "dia_id": "D4:2", + "text": "Hey, Caroline! Nice to hear from you! Love the necklace, any special meaning to it?" + }, + { + "speaker": "Caroline", + "dia_id": "D4:3", + "text": "Thanks, Melanie! This necklace is super special to me - a gift from my grandma in my home country, Sweden. She gave it to me when I was young, and it stands for love, faith and strength. It's like a reminder of my roots and all the love and support I get from my family." + }, + { + "speaker": "Melanie", + "blip_caption": "a photo of a stack of bowls with different designs on them", + "dia_id": "D4:4", + "text": "That's gorgeous, Caroline! It's awesome what items can mean so much to us, right? Got any other objects that you treasure, like that necklace?" + }, + { + "speaker": "Caroline", + "dia_id": "D4:5", + "text": "Yep, Melanie! I've got some other stuff with sentimental value, like my hand-painted bowl. A friend made it for my 18th birthday ten years ago. The pattern and colors are awesome-- it reminds me of art and self-expression." + }, + { + "speaker": "Melanie", + "dia_id": "D4:6", + "text": "That sounds great, Caroline! It's awesome having stuff around that make us think of good connections and times. Actually, I just took my fam camping in the mountains last week - it was a really nice time together!" + }, + { + "speaker": "Caroline", + "dia_id": "D4:7", + "text": "Sounds great, Mel. Glad you made some new family mems. How was it? Anything fun?" + }, + { + "speaker": "Melanie", + "dia_id": "D4:8", + "text": "It was an awesome time, Caroline! We explored nature, roasted marshmallows around the campfire and even went on a hike. The view from the top was amazing! The 2 younger kids love nature. It was so special having these moments together as a family - I'll never forget it!" + }, + { + "speaker": "Caroline", + "dia_id": "D4:9", + "text": "That's awesome, Melanie! Family moments like that are so special. Glad y'all had such a great time." + }, + { + "speaker": "Melanie", + "dia_id": "D4:10", + "text": "Thanks, Caroline! Family time matters to me. What's up with you lately?" + }, + { + "speaker": "Caroline", + "blip_caption": "a photo of a book shelf with many books on it", + "dia_id": "D4:11", + "text": "Lately, I've been looking into counseling and mental health as a career. I want to help people who have gone through the same things as me." + }, + { + "speaker": "Melanie", + "dia_id": "D4:12", + "text": "Sounds great! What kind of counseling and mental health services do you want to persue?" + }, + { + "speaker": "Caroline", + "dia_id": "D4:13", + "text": "I'm still figuring out the details, but I'm thinking of working with trans people, helping them accept themselves and supporting their mental health. Last Friday, I went to an LGBTQ+ counseling workshop and it was really enlightening. They talked about different therapeutic methods and how to best work with trans people. Seeing how passionate these pros were about making a safe space for people like me was amazing." + }, + { + "speaker": "Melanie", + "dia_id": "D4:14", + "text": "Woah, Caroline, it sounds like you're doing some impressive work. It's inspiring to see your dedication to helping others. What motivated you to pursue counseling?" + }, + { + "speaker": "Caroline", + "dia_id": "D4:15", + "text": "Thanks, Melanie. It really mattered. My own journey and the support I got made a huge difference. Now I want to help people go through it too. I saw how counseling and support groups improved my life, so I started caring more about mental health and understanding myself. Now I'm passionate about creating a safe, inviting place for people to grow." + }, + { + "speaker": "Melanie", + "dia_id": "D4:16", + "text": "Wow, Caroline! You've gained so much from your own experience. Your passion and hard work to help others is awesome. Keep it up, you're making a big impact!" + }, + { + "speaker": "Caroline", + "dia_id": "D4:17", + "text": "Thanks, Melanie! Your kind words mean a lot." + }, + { + "speaker": "Melanie", + "blip_caption": "a photo of a book shelf filled with books in a room", + "dia_id": "D4:18", + "text": "Congrats Caroline! Good on you for going after what you really care about." + } + ], + "session_5_date_time": "1:36 pm on 3 July, 2023", + "session_5": [ + { + "speaker": "Caroline", + "dia_id": "D5:1", + "text": "Since we last spoke, some big things have happened. Last week I went to an LGBTQ+ pride parade. Everyone was so happy and it made me feel like I belonged. It showed me how much our community has grown, it was amazing!" + }, + { + "speaker": "Melanie", + "dia_id": "D5:2", + "text": "Wow, Caroline, sounds like the parade was an awesome experience! It's great to see the love and support for the LGBTQ+ community. Congrats! Has this experience influenced your goals at all?" + }, + { + "speaker": "Caroline", + "dia_id": "D5:3", + "text": "Thanks, Mel! It really motivated me for sure. Talking to the community made me want to use my story to help others too - I'm still thinking that counseling and mental health is the way to go. I'm super excited to give back. " + }, + { + "speaker": "Melanie", + "img_url": [ + "https://m.media-amazon.com/images/I/A1uELSr5rgL.jpg" + ], + "blip_caption": "a photo of a person holding a frisbee in their hand", + "query": "family frisbee game", + "dia_id": "D5:4", + "text": "Wow, Caroline! That's great! I just signed up for a pottery class yesterday. It's like therapy for me, letting me express myself and get creative. Have you found any activities that make you feel the same way?" + }, + { + "speaker": "Caroline", + "dia_id": "D5:5", + "text": "Wow, Melanie! I'm getting creative too, just learning the piano. What made you try pottery?" + }, + { + "speaker": "Melanie", + "img_url": [ + "https://therusticbarnct.com/cdn/shop/files/image_05483f46-4845-433b-a4cf-0fc61fe1aa79.jpg" + ], + "blip_caption": "a photo of a bowl with a black and white flower design", + "query": "pottery painted bowl intricate design", + "dia_id": "D5:6", + "text": "I'm a big fan of pottery - the creativity and skill is awesome. Plus, making it is so calming. Look at this!" + }, + { + "speaker": "Caroline", + "dia_id": "D5:7", + "text": "That bowl is gorgeous! The black and white design looks so fancy. Did you make it?" + }, + { + "speaker": "Melanie", + "dia_id": "D5:8", + "text": "Thanks, Caroline! Yeah, I made this bowl in my class. It took some work, but I'm pretty proud of it." + }, + { + "speaker": "Caroline", + "dia_id": "D5:9", + "text": "Nice job! You really put in the work and it definitely shows. Your creativity looks great!" + }, + { + "speaker": "Melanie", + "dia_id": "D5:10", + "text": "Thanks, Caroline! Your kind words mean a lot. Pottery is a huge part of my life, not just a hobby - it helps me express my emotions. Clay is incredible, it brings me so much joy!" + }, + { + "speaker": "Caroline", + "dia_id": "D5:11", + "text": "Wow, Mel, I'm so stoked for you that art is helping you express yourself and bring you joy! Keep it up!" + }, + { + "speaker": "Melanie", + "dia_id": "D5:12", + "text": "Thanks, Caroline! I'm excited to see where pottery takes me. Anything coming up you're looking forward to?" + }, + { + "speaker": "Caroline", + "dia_id": "D5:13", + "text": "Thanks Mel! I'm going to a transgender conference this month. I'm so excited to meet other people in the community and learn more about advocacy. It's gonna be great!" + }, + { + "speaker": "Melanie", + "dia_id": "D5:14", + "text": "Sounds awesome, Caroline! Have a great time and learn a lot. Have fun!" + }, + { + "speaker": "Caroline", + "dia_id": "D5:15", + "text": "Cool, thanks Mel! Can't wait. I'll keep ya posted. Bye!" + }, + { + "speaker": "Melanie", + "dia_id": "D5:16", + "text": "Bye, Caroline! Can't wait to hear about it. Have fun and stay safe!" + } + ], + "session_6_date_time": "8:18 pm on 6 July, 2023", + "session_6": [ + { + "speaker": "Caroline", + "dia_id": "D6:1", + "text": "Hey Mel! Long time no talk. Lots has been going on since then!" + }, + { + "speaker": "Melanie", + "dia_id": "D6:2", + "text": "Hey Caroline! Missed you. Anything new? Spill the beans!" + }, + { + "speaker": "Caroline", + "dia_id": "D6:3", + "text": "Since our last chat, I've been looking into counseling or mental health work more. I'm passionate about helping people and making a positive impact. It's tough, but really rewarding too. Anything new happening with you?" + }, + { + "speaker": "Melanie", + "img_url": [ + "https://live.staticflickr.com/3201/2867258131_2d8bc22859_b.jpg" + ], + "blip_caption": "a photography of two children playing in a water play area", + "query": "kids laughing dinosaur exhibit museum", + "dia_id": "D6:4", + "re-download": true, + "text": "That's awesome, Caroline! Congrats on following your dreams. Yesterday I took the kids to the museum - it was so cool spending time with them and seeing their eyes light up!" + }, + { + "speaker": "Caroline", + "dia_id": "D6:5", + "text": "Melanie, that's a great pic! That must have been awesome. What were they so stoked about?" + }, + { + "speaker": "Melanie", + "dia_id": "D6:6", + "text": "They were stoked for the dinosaur exhibit! They love learning about animals and the bones were so cool. It reminds me why I love being a mom." + }, + { + "speaker": "Caroline", + "img_url": [ + "https://i.pinimg.com/originals/02/94/c3/0294c3460b66d1fd50530e4bd5a2e1f5.jpg" + ], + "blip_caption": "a photo of a bookcase filled with books and toys", + "query": "bookshelf childrens books library", + "dia_id": "D6:7", + "text": "Being a mom is awesome. I'm creating a library for when I have kids. I'm really looking forward to reading to them and opening up their minds." + }, + { + "speaker": "Melanie", + "dia_id": "D6:8", + "text": "Sounds great! What kind of books you got in your library?" + }, + { + "speaker": "Caroline", + "dia_id": "D6:9", + "text": "I've got lots of kids' books- classics, stories from different cultures, educational books, all of that. What's a favorite book you remember from your childhood?" + }, + { + "speaker": "Melanie", + "img_url": [ + "http://bookworm-detective.myshopify.com/cdn/shop/products/PXL_20210428_222022427.jpg" + ], + "blip_caption": "a photo of a book cover with a picture of a girl and a cat", + "query": "charlotte's web book", + "dia_id": "D6:10", + "text": "I loved reading \"Charlotte's Web\" as a kid. It was so cool seeing how friendship and compassion can make a difference." + }, + { + "speaker": "Caroline", + "img_url": [ + "https://i.pinimg.com/originals/41/d5/60/41d5601e4ab0959ce5e29683a2660938.jpg" + ], + "blip_caption": "a photo of a group of women sitting on a blanket in a park", + "query": "group friends picnic", + "dia_id": "D6:11", + "text": "Wow, that's great! It sure shows how important friendship and compassion are. It's made me appreciate how lucky I am to have my friends and family helping with my transition. They make all the difference. We even had a picnic last week!" + }, + { + "speaker": "Melanie", + "dia_id": "D6:12", + "text": "That's a gorgeous photo, Caroline! Wow, the love around you is awesome. How have your friends and fam been helping you out with your transition?" + }, + { + "speaker": "Caroline", + "dia_id": "D6:13", + "text": "Thanks, Melanie! This support network has been amazing. They've been there for me every step of the way giving me love, guidance, and acceptance. I couldn't have done it without them." + }, + { + "speaker": "Melanie", + "dia_id": "D6:14", + "text": "Wow, Caroline! It's great you have people to support you, that's really awesome!" + }, + { + "speaker": "Caroline", + "dia_id": "D6:15", + "text": "I'm so lucky to have such a great support system around me. Their love and encouragement has really helped me accept and grow into my true self. They've been instrumental in my transition." + }, + { + "speaker": "Melanie", + "img_url": [ + "https://i.redd.it/ye1cp24b18w01.jpg" + ], + "blip_caption": "a photo of a family sitting around a campfire on the beach", + "query": "family campfire", + "dia_id": "D6:16", + "text": "Glad you have support, Caroline! Unconditional love is so important. Here's a pic of my family camping at the beach. We love it, it brings us closer!" + } + ], + "session_7_date_time": "4:33 pm on 12 July, 2023", + "session_7": [ + { + "speaker": "Caroline", + "dia_id": "D7:1", + "text": "Hey Mel, great to chat with you again! So much has happened since we last spoke - I went to an LGBTQ conference two days ago and it was really special. I got the chance to meet and connect with people who've gone through similar journeys. It was such a welcoming environment and I felt totally accepted. I'm really thankful for this amazing community - it's shown me how important it is to fight for trans rights and spread awareness." + }, + { + "speaker": "Melanie", + "dia_id": "D7:2", + "text": "Wow, Caroline, that sounds awesome! So glad you felt accepted and supported. Events like these are great for reminding us of how strong community can be!" + }, + { + "speaker": "Caroline", + "dia_id": "D7:3", + "text": "Yeah, it's true! Having people who back you makes such a huge difference. It's great to see how far LGBTQ rights have come, but there's still plenty of progress to be made. I wanna help make a difference." + }, + { + "speaker": "Melanie", + "dia_id": "D7:4", + "text": "Wow, Caroline. We've come so far, but there's more to do. Your drive to help is awesome! What's your plan to pitch in?" + }, + { + "speaker": "Caroline", + "dia_id": "D7:5", + "text": "Thanks, Mell! I'm still looking into counseling and mental health jobs. It's important to me that people have someone to talk to, and I want to help make that happen." + }, + { + "speaker": "Melanie", + "dia_id": "D7:6", + "text": "Wow, Caroline! You're so inspiring for wanting to help others with their mental health. What's pushing you to keep going forward with it?" + }, + { + "speaker": "Caroline", + "dia_id": "D7:7", + "text": "I struggled with mental health, and support I got was really helpful. It made me realize how important it is for others to have a support system. So, I started looking into counseling and mental health career options, so I could help other people on their own journeys like I was helped." + }, + { + "speaker": "Melanie", + "img_url": [ + "https://www.speakers.co.uk/microsites/tom-oliver/wp-content/uploads/2014/11/Book-Cover-3D1.jpg" + ], + "blip_caption": "a photography of a book cover with a gold coin on it", + "query": "painted canvas follow your dreams", + "dia_id": "D7:8", + "re-download": true, + "text": "Caroline, so glad you got the support! Your experience really brought you to where you need to be. You're gonna make a huge difference! This book I read last year reminds me to always pursue my dreams, just like you are doing!🌟" + }, + { + "speaker": "Caroline", + "dia_id": "D7:9", + "text": "Thanks so much, Mel! Seeing this pic just made me appreciate my love of reading even more. Books guide me, motivate me and help me discover who I am. They're a huge part of my journey, and this one's reminding me to keep going and never give up!" + }, + { + "speaker": "Melanie", + "dia_id": "D7:10", + "text": "Wow, Caroline! Books have such an awesome power! Which one has been your favorite guide?" + }, + { + "speaker": "Caroline", + "img_url": [ + "https://m.media-amazon.com/images/I/A1CPpaLFR2L.jpg" + ], + "blip_caption": "a photo of a dog sitting in a boat on the water", + "query": "becoming nicole book amy ellis nutt", + "dia_id": "D7:11", + "text": "I loved \"Becoming Nicole\" by Amy Ellis Nutt. It's a real inspiring true story about a trans girl and her family. It made me feel connected and gave me a lot of hope for my own path. Highly recommend it for sure!" + }, + { + "speaker": "Melanie", + "dia_id": "D7:12", + "text": "That sounds awesome! What did you take away from it to use in your life?" + }, + { + "speaker": "Caroline", + "dia_id": "D7:13", + "text": "It taught me self-acceptance and how to find support. It also showed me that tough times don't last - hope and love exist. Pets bring so much joy too, though." + }, + { + "speaker": "Melanie", + "img_url": [ + "https://st3.depositphotos.com/12674628/16006/i/1600/depositphotos_160060676-stock-photo-multiethnic-girls-with-puppy.jpg" + ], + "blip_caption": "a photography of two little girls sitting on the steps with a dog", + "query": "daughters playing with pet dog backyard", + "dia_id": "D7:14", + "re-download": true, + "text": "Caroline, those lessons are great - self-acceptance and finding support are key. Plus pets are awesome for joy and comfort, can't agree more! " + }, + { + "speaker": "Caroline", + "dia_id": "D7:15", + "text": "That's so nice! What pet do you have?" + }, + { + "speaker": "Melanie", + "img_url": [ + "https://i.redd.it/u26t78f0idd91.jpg" + ], + "blip_caption": "a photo of a cat laying on the floor with its head on the floor", + "query": "dog cat kids playing joy", + "dia_id": "D7:16", + "text": "We've got a pup and a kitty. That's the dog, and here's our cat! They brighten up our day and always make us smile." + }, + { + "speaker": "Caroline", + "dia_id": "D7:17", + "text": "Ah, they're adorable! What are their names? Pets sure do bring so much joy to us!" + }, + { + "speaker": "Melanie", + "img_url": [ + "https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEhwGRNI2jDkrALJHgL2LWfW2rUGhN-GA4OL_gXU2fHPyxtst2MPrv9hkyOMdpj5SppLNYiQrcXUUq90vv5es8ueswy2tuu0Lqa2lh2vKOfDZ5SXSdLVMVvBrfLbFJG19QiqDbv1xs38fv-atd4MYOesJ4c89sQTzv6k93PDQ5T0dwVJV9O2FF95woyP3Q/s4032/IMG_9747.jpg" + ], + "blip_caption": "a photo of a person wearing pink sneakers on a white rug", + "query": "purple running shoe", + "dia_id": "D7:18", + "text": "Luna and Oliver! They are so sweet and playful - they really liven up the house! Just got some new shoes, too!" + }, + { + "speaker": "Caroline", + "dia_id": "D7:19", + "text": "Love that purple color! For walking or running?" + }, + { + "speaker": "Melanie", + "blip_caption": "a photo of a pair of pink sneakers in a box", + "dia_id": "D7:20", + "text": "Thanks, Caroline! These are for running. Been running longer since our last chat - a great way to destress and clear my mind." + }, + { + "speaker": "Caroline", + "dia_id": "D7:21", + "text": "Wow! What got you into running?" + }, + { + "speaker": "Melanie", + "dia_id": "D7:22", + "text": "I've been running farther to de-stress, which has been great for my headspace." + }, + { + "speaker": "Caroline", + "dia_id": "D7:23", + "text": "Cool, Melanie! Running can really boost your mood. Keep it up!" + }, + { + "speaker": "Melanie", + "dia_id": "D7:24", + "text": "Thanks, Caroline! This has been great for my mental health. I'm gonna keep it up." + }, + { + "speaker": "Caroline", + "dia_id": "D7:25", + "text": "Awesome, Melanie! Mental health's a priority, so make sure you take care of yourself." + }, + { + "speaker": "Melanie", + "dia_id": "D7:26", + "text": "Caroline, thanks! Mental health is important to me, and it's made such an improvement!" + }, + { + "speaker": "Caroline", + "dia_id": "D7:27", + "text": "Glad it helped ya, Melanie!" + } + ], + "session_8_date_time": "1:51 pm on 15 July, 2023", + "session_8": [ + { + "speaker": "Caroline", + "dia_id": "D8:1", + "text": "Hey Mel, what's up? Been a busy week since we talked." + }, + { + "speaker": "Melanie", + "img_url": [ + "https://images.rawpixel.com/image_social_landscape/cHJpdmF0ZS9sci9pbWFnZXMvd2Vic2l0ZS8yMDIyLTExL2ZsNDg2NDgxOTYyMDMtaW1hZ2UuanBn.jpg" + ], + "blip_caption": "a photography of a group of children making clay sculptures in a classroom", + "query": "pottery workshop family making clay pots", + "dia_id": "D8:2", + "re-download": true, + "text": "Hey Caroline, it's been super busy here. So much since we talked! Last Fri I finally took my kids to a pottery workshop. We all made our own pots, it was fun and therapeutic!" + }, + { + "speaker": "Caroline", + "dia_id": "D8:3", + "text": "Wow, Mel! Sounds like you and the kids had a blast. How'd they like it?" + }, + { + "speaker": "Melanie", + "img_url": [ + "https://monstermonster.shop/cdn/shop/products/mug-class_5000x.jpg" + ], + "blip_caption": "a photo of a cup with a dog face on it", + "query": "kids pottery finished pieces", + "dia_id": "D8:4", + "text": "The kids loved it! They were so excited to get their hands dirty and make something with clay. It was special to watch their creativity and imagination come to life, they made this!" + }, + { + "speaker": "Caroline", + "dia_id": "D8:5", + "text": "Aww, that's so sweet! That cup is so cute. It's awesome to see how kids show their personalities through art. What other creative projects do you do with them, besides pottery?" + }, + { + "speaker": "Melanie", + "img_url": [ + "https://i.pinimg.com/originals/ea/d9/d7/ead9d79b58ca80a38a744b5ab70482db.jpg" + ], + "blip_caption": "a photo of a painting of a sunset with a palm tree", + "query": "painting vibrant flowers sunset sky", + "dia_id": "D8:6", + "text": "We love painting together lately, especially nature-inspired ones. Here's our latest work from last weekend." + }, + { + "speaker": "Caroline", + "dia_id": "D8:7", + "text": "Wow Mel, that painting's amazing! The colors are so bold and it really highlights the beauty of nature. Y'all work on it together?" + }, + { + "speaker": "Melanie", + "img_url": [ + "https://karengimson.files.wordpress.com/2017/06/img_7222.jpg" + ], + "blip_caption": "a photo of a field of purple flowers with green leaves", + "query": "path lined purple flowers nature", + "dia_id": "D8:8", + "text": "Thanks, Caroline! We both helped with the painting - it was great bonding over it and chatting about nature. We found these lovely flowers. Appreciating the small things in life, too." + }, + { + "speaker": "Caroline", + "dia_id": "D8:9", + "text": "That photo is stunning! So glad you bonded over our love of nature. Last Friday I went to a council meeting for adoption. It was inspiring and emotional - so many people wanted to create loving homes for children in need. It made me even more determined to adopt." + }, + { + "speaker": "Melanie", + "img_url": [ + "https://assets.eflorist.com/assets/products/PHR_/TEV57-5A.jpg" + ], + "blip_caption": "a photo of a blue vase with a bouquet of sunflowers and roses", + "query": "sunflower bouquet", + "dia_id": "D8:10", + "text": "Wow, Caroline, way to go! Your future fam will get a kick out of having you. What do you think of these?" + }, + { + "speaker": "Caroline", + "dia_id": "D8:11", + "text": "Thanks Melanie - love the blue vase in the pic! Blue's my fave, it makes me feel relaxed. Sunflowers mean warmth and happiness, right? While roses stand for love and beauty? That's neat. What do flowers mean to you?" + }, + { + "speaker": "Melanie", + "img_url": [ + "https://blueblossomrentals.com/cdn/shop/products/image_909fb96b-4208-429b-9a6f-59dffa3cb546.jpg" + ], + "blip_caption": "a photo of a row of white chairs with flowers on them", + "query": "garden full of flowers wedding decorations", + "dia_id": "D8:12", + "text": "Flowers bring joy. They represent growth, beauty and reminding us to appreciate the small moments. They were an important part of my wedding decor and always remind me of that day." + }, + { + "speaker": "Caroline", + "dia_id": "D8:13", + "text": "It must have been special at your wedding. I wish I had known you back then!" + }, + { + "speaker": "Melanie", + "img_url": [ + "https://platinumnotary.files.wordpress.com/2023/03/img-6679.jpg" + ], + "blip_caption": "a photo of a wedding ceremony in a greenhouse with people taking pictures", + "query": "wedding ceremony", + "dia_id": "D8:14", + "text": "It was amazing, Caroline. The day was full of love and joy. Everyone we love was there to celebrate us - it was really special." + }, + { + "speaker": "Caroline", + "dia_id": "D8:15", + "text": "Wow, what a great day! Glad everyone could make it. What was your favorite part?" + }, + { + "speaker": "Melanie", + "img_url": [ + "https://s3-us-west-2.amazonaws.com/amm-prod/wedding_photos/photos/000/024/198/original/4B873921-0596-4A6B-8CD8-C6E5C2B024AF.png" + ], + "blip_caption": "a photo of a man and woman standing on a beach", + "query": "vows partner holding hands ceremony", + "dia_id": "D8:16", + "text": "Marrying my partner and promising to be together forever was the best part." + }, + { + "speaker": "Caroline", + "img_url": [ + "https://dynaimage.cdn.cnn.com/cnn/digital-images/org/dfc95f14-b325-431c-b977-5b6dc2d35f9c.jpg" + ], + "blip_caption": "a photo of a parade with people walking down the street", + "query": "rainbow flag pride march", + "dia_id": "D8:17", + "text": "Wow, nice pic! You both looked amazing. One special memory for me was this pride parade I went to a few weeks ago." + }, + { + "speaker": "Melanie", + "dia_id": "D8:18", + "text": "Wow, looks awesome! Did you join in?" + }, + { + "speaker": "Caroline", + "img_url": [ + "http://ninalemsparty.com/cdn/shop/collections/iStock-1292280203.jpg" + ], + "blip_caption": "a photo of a group of people holding up signs and smiling", + "query": "lgbtq+ pride parade vibrant flags smiling faces", + "dia_id": "D8:19", + "text": "Yes, I did. It was amazing! I felt so accepted and happy, just being around people who accepted and celebrated me. It's definitely a top memory." + }, + { + "speaker": "Melanie", + "dia_id": "D8:20", + "text": "Wow, what an experience! How did it make you feel?" + }, + { + "speaker": "Caroline", + "blip_caption": "a photo of a rainbow flag on a pole on a carpet", + "dia_id": "D8:21", + "text": "I felt so proud and grateful - the vibes were amazing and it was comforting to know I'm not alone and have a great community around me." + }, + { + "speaker": "Melanie", + "dia_id": "D8:22", + "text": "Wow, Caroline! That's huge! How did it feel to be around so much love and acceptance?" + }, + { + "speaker": "Caroline", + "blip_caption": "a photo of a group of people sitting on the ground with a dog", + "dia_id": "D8:23", + "text": "It was awesome, Melanie! Being around people who embrace and back me up is beyond words. It really inspired me." + }, + { + "speaker": "Melanie", + "blip_caption": "a photo of a girl sitting in a teepee with stuffed animals", + "dia_id": "D8:24", + "text": "Wow, that sounds awesome! Your friends and community really have your back. What's been the best part of it?" + }, + { + "speaker": "Caroline", + "blip_caption": "a photo of a teepee with a teddy bear and pillows", + "dia_id": "D8:25", + "text": "Realizing I can be me without fear and having the courage to transition was the best part. It's so freeing to express myself authentically and have people back me up." + }, + { + "speaker": "Melanie", + "blip_caption": "a photo of a buddha statue and a candle on a table", + "dia_id": "D8:26", + "text": "That's awesome, Caro! You've found the courage to be yourself - that's important for our mental health and finding peace." + }, + { + "speaker": "Caroline", + "dia_id": "D8:27", + "text": "Thanks, Melanie! Been a long road, but I'm proud of how far I've come. How're you doing finding peace?" + }, + { + "speaker": "Melanie", + "blip_caption": "a photo of a man holding a frisbee in front of a frisbee golf basket", + "dia_id": "D8:28", + "text": "I'm getting there, Caroline. Creativity and family keep me at peace." + }, + { + "speaker": "Caroline", + "dia_id": "D8:29", + "text": "That's awesome, Melanie! How have your family been supportive during your move?" + }, + { + "speaker": "Melanie", + "dia_id": "D8:30", + "text": "My fam's been awesome - they helped out and showed lots of love and support." + }, + { + "speaker": "Caroline", + "dia_id": "D8:31", + "text": "Wow, Mel, family love and support is the best!" + }, + { + "speaker": "Melanie", + "img_url": [ + "http://cragmama.com/wp-content/uploads//2016/10/IMG_4568.jpg" + ], + "blip_caption": "a photo of a man and two children sitting around a campfire", + "query": "family camping trip roasting marshmallows campfire", + "dia_id": "D8:32", + "text": "Yeah, Caroline, my family's been great - their love and support really helped me through tough times. It's awesome! We even went on another camping trip in the forest." + }, + { + "speaker": "Caroline", + "blip_caption": "a photo of a family walking through a forest with a toddler", + "dia_id": "D8:33", + "text": "Awesome, Mel! Family support's huge. What else do you guys like doing together?" + }, + { + "speaker": "Melanie", + "dia_id": "D8:34", + "text": "We enjoy hiking in the mountains and exploring forests. It's a cool way to connect with nature and each other." + }, + { + "speaker": "Caroline", + "dia_id": "D8:35", + "text": "Wow, Mel, that sounds awesome! Exploring nature and family time is so special." + }, + { + "speaker": "Melanie", + "dia_id": "D8:36", + "text": "Yeah, Caroline, they're some of my fave memories. It brings us together and brings us happiness. Glad you're here to share in it." + }, + { + "speaker": "Caroline", + "dia_id": "D8:37", + "text": "Thanks, Melanie! Really glad to have you as a friend to share my journey. You're awesome!" + }, + { + "speaker": "Melanie", + "dia_id": "D8:38", + "text": "Thanks, Caroline! Appreciate your friendship. It's great to have a supporter!" + }, + { + "speaker": "Caroline", + "dia_id": "D8:39", + "text": "No worries, Mel! Your friendship means so much to me. Enjoy your day!" + } + ], + "session_9_date_time": "2:31 pm on 17 July, 2023", + "session_9": [ + { + "speaker": "Melanie", + "dia_id": "D9:1", + "text": "Hey Caroline, hope all's good! I had a quiet weekend after we went camping with my fam two weekends ago. It was great to unplug and hang with the kids. What've you been up to? Anything fun over the weekend?" + }, + { + "speaker": "Caroline", + "dia_id": "D9:2", + "text": "Hey Melanie! That sounds great! Last weekend I joined a mentorship program for LGBTQ youth - it's really rewarding to help the community." + }, + { + "speaker": "Melanie", + "dia_id": "D9:3", + "text": "Wow, Caroline! It's great that you're helping out. How's it going? Got any cool experiences you can share?" + }, + { + "speaker": "Caroline", + "dia_id": "D9:4", + "text": "The mentoring is going great! I've met some amazing young folks and supported them along the way. It's inspiring to see how resilient and strong they are." + }, + { + "speaker": "Melanie", + "dia_id": "D9:5", + "text": "Wow, Caroline, that sounds super rewarding! Young people's resilience is amazing. Care to share some stories?" + }, + { + "speaker": "Caroline", + "dia_id": "D9:6", + "text": "I mentor a transgender teen just like me. We've been working on building up confidence and finding positive strategies, and it's really been paying off! We had a great time at the LGBT pride event last month." + }, + { + "speaker": "Melanie", + "dia_id": "D9:7", + "text": "Caroline, awesome news that you two are getting along! What was it like for you both? Care to fill me in?" + }, + { + "speaker": "Caroline", + "img_url": [ + "https://res.cloudinary.com/dragonspell/images/w_1440,h_864,c_fill,dpr_auto,fl_progressive:steep,f_auto/w_1440,h_864/v1571420662/www.travelportland.com/Portland-Pride-Parade-Downtown/Portland-Pride-Parade-Downtown.jpg" + ], + "blip_caption": "a photo of a woman holding a rainbow umbrella in the air", + "query": "lgbt pride event", + "dia_id": "D9:8", + "text": "The pride event was awesome! It was so encouraging to be surrounded by so much love and acceptance." + }, + { + "speaker": "Melanie", + "dia_id": "D9:9", + "text": "Wow! What's the best part you remember from it?" + }, + { + "speaker": "Caroline", + "dia_id": "D9:10", + "text": "Seeing my mentee's face light up when they saw the support was the best! Such a special moment." + }, + { + "speaker": "Melanie", + "dia_id": "D9:11", + "text": "Wow, Caroline! They must have felt so appreciated. It's awesome to see the difference we can make in each other's lives. Any other exciting LGBTQ advocacy stuff coming up?" + }, + { + "speaker": "Caroline", + "dia_id": "D9:12", + "text": "Yay! Next month I'm having an LGBTQ art show with my paintings - can't wait!" + }, + { + "speaker": "Melanie", + "blip_caption": "a photo of a painting with a blue and yellow design", + "dia_id": "D9:13", + "text": "Wow, Caroline, that sounds awesome! Can't wait to see your art - got any previews?" + }, + { + "speaker": "Caroline", + "img_url": [ + "https://images.fineartamerica.com/images/artworkimages/mediumlarge/1/abstract-landscape-bold-colorful-painting-megan-duncanson.jpg" + ], + "blip_caption": "a photography of a painting of a tree with a bright sun in the background", + "query": "preview painting art show", + "dia_id": "D9:14", + "re-download": true, + "text": "Check out my painting for the art show! Hope you like it." + }, + { + "speaker": "Melanie", + "dia_id": "D9:15", + "text": "Wow, Caroline, that painting is awesome! Those colors are so vivid and the whole thing looks really unified. What inspired you?" + }, + { + "speaker": "Caroline", + "dia_id": "D9:16", + "text": "Thanks, Melanie! I painted this after I visited a LGBTQ center. I wanted to capture everyone's unity and strength." + }, + { + "speaker": "Melanie", + "dia_id": "D9:17", + "text": "Wow, Caroline! It really conveys unity and strength - such a gorgeous piece! My kids and I just finished another painting like our last one." + } + ], + "session_10_date_time": "8:56 pm on 20 July, 2023", + "session_10": [ + { + "speaker": "Caroline", + "dia_id": "D10:1", + "text": "Hey Melanie! Just wanted to say hi!" + }, + { + "speaker": "Melanie", + "dia_id": "D10:2", + "text": "Hey Caroline! Good to talk to you again. What's up? Anything new since last time?" + }, + { + "speaker": "Caroline", + "dia_id": "D10:3", + "text": "Hey Mel! A lot's happened since we last chatted - I just joined a new LGBTQ activist group last Tues. I'm meeting so many cool people who are as passionate as I am about rights and community support. I'm giving my voice and making a real difference, plus it's fulfilling in so many ways. It's just great, you know?" + }, + { + "speaker": "Melanie", + "dia_id": "D10:4", + "text": "That's awesome, Caroline! Glad to hear you found a great group where you can have an impact. Bet it feels great to be able to speak your truth and stand up for what's right. Want to tell me a bit more about it?" + }, + { + "speaker": "Caroline", + "dia_id": "D10:5", + "text": "Thanks, Melanie! It's awesome to have our own platform to be ourselves and support others' rights. Our group, 'Connected LGBTQ Activists', is made of all kinds of people investing in positive changes. We have regular meetings, plan events and campaigns, to get together and support each other." + }, + { + "speaker": "Melanie", + "dia_id": "D10:6", + "text": "Wow, Caroline, your group sounds awesome! Supporting each other and making good things happen - that's so inspiring! Have you been part of any events or campaigns lately?" + }, + { + "speaker": "Caroline", + "dia_id": "D10:7", + "text": "Last weekend our city held a pride parade! So many people marched through the streets waving flags, holding signs and celebrating love and diversity. I missed it but it was a powerful reminder that we are not alone in this fight for equality and inclusivity. Change is possible!" + }, + { + "speaker": "Melanie", + "img_url": [ + "https://mdkidadventures.files.wordpress.com/2023/06/img_2130.jpg" + ], + "blip_caption": "a photo of three children playing on the beach with a kite", + "query": "beach family playing frisbee sandy shore", + "dia_id": "D10:8", + "text": "Wow, fantastic, Caroline! Bet the atmosphere was incredible. Oh yeah, we went to the beach recently. It was awesome! The kids had such a blast." + }, + { + "speaker": "Caroline", + "dia_id": "D10:9", + "text": "Sounds fun! What was the best part? Do you do it often with the kids?" + }, + { + "speaker": "Melanie", + "blip_caption": "a photo of a sand castle on the beach with a blue sky", + "dia_id": "D10:10", + "text": "Seeing my kids' faces so happy at the beach was the best! We don't go often, usually only once or twice a year. But those times are always special to spend time together and chill." + }, + { + "speaker": "Caroline", + "dia_id": "D10:11", + "text": "Sounds special, those beach trips! Do you have any other summer traditions you all do together? Create those memories!" + }, + { + "speaker": "Melanie", + "img_url": [ + "https://i.redd.it/hjh0wp8s721a1.jpg" + ], + "blip_caption": "a photo of a fire pit with a lot of fire and sparks", + "query": "family camping trip campfire night", + "dia_id": "D10:12", + "text": "We always look forward to our family camping trip. We roast marshmallows, tell stories around the campfire and just enjoy each other's company. It's the highlight of our summer!" + }, + { + "speaker": "Caroline", + "dia_id": "D10:13", + "text": "Wow, Mel, that's awesome! What's your best camping memory?" + }, + { + "speaker": "Melanie", + "img_url": [ + "https://i.redd.it/ms0tvo85cto91.jpg" + ], + "blip_caption": "a photo of a plane flying in the sky with a star filled sky", + "query": "shooting star night sky", + "dia_id": "D10:14", + "text": "I'll always remember our camping trip last year when we saw the Perseid meteor shower. It was so amazing lying there and watching the sky light up with streaks of light. We all made wishes and felt so at one with the universe. That's a memory I'll never forget." + }, + { + "speaker": "Caroline", + "dia_id": "D10:15", + "text": "Cool! What did it look like?" + }, + { + "speaker": "Melanie", + "img_url": [ + "https://i.redd.it/eqtu6adwcrfb1.jpg" + ], + "blip_caption": "a photo of a plane flying in the sky with a trail of smoke coming out of it", + "query": "night sky stars meteor shower", + "dia_id": "D10:16", + "text": "The sky was so clear and filled with stars, and the meteor shower was amazing - it felt like we were part of something huge and awe-inspiring." + }, + { + "speaker": "Caroline", + "dia_id": "D10:17", + "text": "Wow, Mel. That must've been breathtaking!" + }, + { + "speaker": "Melanie", + "blip_caption": "a photo of a beach with footprints in the sand and a blue sky", + "dia_id": "D10:18", + "text": "It was one of those moments where I felt tiny and in awe of the universe. Reminds me how awesome life is - so many little moments like that." + }, + { + "speaker": "Caroline", + "dia_id": "D10:19", + "text": "That's great, Mel! What other good memories do you have that make you feel thankful for life?" + }, + { + "speaker": "Melanie", + "dia_id": "D10:20", + "text": "I'll never forget the day my youngest took her first steps. Seeing her wobble as she took those initial steps really put into perspective how fleeting life is and how lucky I am to be able to share these moments." + }, + { + "speaker": "Caroline", + "blip_caption": "a photo of a baby in a white crib with a blue blanket", + "dia_id": "D10:21", + "text": "Aw, that's sweet, Mel! Those milestones are great reminders of how special our bonds are." + }, + { + "speaker": "Melanie", + "img_url": [ + "https://freerangestock.com/sample/134391/happy-family-holding-hands-with-ocean-and-sunset-in-the-background.jpg" + ], + "blip_caption": "a photography of a family standing on the beach at sunset", + "query": "children playing and laughing", + "dia_id": "D10:22", + "re-download": true, + "text": "Yeah, they sure are. It's special moments like these that make me appreciate life and how lucky I am to be with my family and have our love." + }, + { + "speaker": "Caroline", + "dia_id": "D10:23", + "text": "Wow, Melanie, what a beautiful moment! Lucky you to have such an awesome family!" + }, + { + "speaker": "Melanie", + "dia_id": "D10:24", + "text": "Thanks, Caroline! I'm really lucky to have my family; they bring so much joy and love." + } + ], + "session_11_date_time": "2:24 pm on 14 August, 2023", + "session_11": [ + { + "speaker": "Melanie", + "dia_id": "D11:1", + "text": "Hey Caroline! Last night was amazing! We celebrated my daughter's birthday with a concert surrounded by music, joy and the warm summer breeze. Seeing my kids' smiles was so awesome, and I'm so thankful for our special moments together." + }, + { + "speaker": "Caroline", + "blip_caption": "a photo of a poster for a concert with a picture of a man", + "dia_id": "D11:2", + "text": "Wow, sounds wonderful! Your love for your kids is so awesome. What concert was it? The advocacy event was a cool experience - so much love and support, amazing!" + }, + { + "speaker": "Melanie", + "dia_id": "D11:3", + "text": "Thanks, Caroline! It was Matt Patterson, he is so talented! His voice and songs were amazing. What's up with you? Anything interesting going on?" + }, + { + "speaker": "Caroline", + "dia_id": "D11:4", + "text": "Wow, Mel, glad you had a blast at the concert. A lot's happened since we talked. I went to a pride parade last Friday and it was awesome - so much energy and love everywhere. Really made me proud and reminded me how important it is to keep standing up for equality." + }, + { + "speaker": "Melanie", + "blip_caption": "a photo of a band performing on stage with a sign that says all are welcome", + "dia_id": "D11:5", + "text": "Wow, that's awesome! How did it feel being part of that community?" + }, + { + "speaker": "Caroline", + "img_url": [ + "https://cloudfront-us-east-1.images.arcpublishing.com/opb/35SV3NIC4ZBRTLDGHUJ5QWU5WY.jpg" + ], + "blip_caption": "a photo of a group of people walking down a street with balloons", + "query": "pride parade crowd", + "dia_id": "D11:6", + "text": "It was so inspiring, Mel! Check out the crowd. People of all kinds celebrating love and acceptance - it really pushed me to keep fighting for LGBTQ rights." + }, + { + "speaker": "Melanie", + "img_url": [ + "https://livingmividaloca.com/wp-content/uploads/2023/06/anaheim-town-square-concert.jpg" + ], + "blip_caption": "a photo of a group of people sitting on chairs watching a band", + "query": "outdoor concert family loving accepting environment", + "dia_id": "D11:7", + "text": "Wow, Caroline! That sounds awesome. This pic's from last night - looks like everyone was having a blast! Reminds me it's important to cultivate a loving and accepting environment for our kids. How do you stay inclusive in your work as an artist?" + }, + { + "speaker": "Caroline", + "img_url": [ + "https://www.dawnsilerart.com/wp-content/uploads/sites/3130/2020/11/YCNHTMC-CU9.jpg" + ], + "blip_caption": "a photo of a painting with a painting brush and paint on it", + "query": "painting vibrant colors diverse representation", + "dia_id": "D11:8", + "text": "That pic is cool! Representing inclusivity and diversity in my art is important to me. I also use it to speak up for the LGBTQ+ community and push for acceptance. Here's a recent painting!" + }, + { + "speaker": "Melanie", + "dia_id": "D11:9", + "text": "Wow, that rocks! What's the main idea of your art?" + }, + { + "speaker": "Caroline", + "blip_caption": "a photo of a painting of a woman with a cow in her lap", + "dia_id": "D11:10", + "text": "My art is about expressing my trans experience. It's my way of showing my story and helping people understand the trans community." + }, + { + "speaker": "Melanie", + "blip_caption": "a photo of a person holding a purple bowl in their hand", + "dia_id": "D11:11", + "text": "Your art's amazing, Caroline. I love how you use it to tell your stories and teach people about trans folks. I'd love to see another painting of yours!" + }, + { + "speaker": "Caroline", + "img_url": [ + "https://media.artsper.com/artwork/2013795_1_l.jpg" + ], + "blip_caption": "a photo of a painting of a woman with a red shirt", + "query": "painting embracing identity purple blue", + "dia_id": "D11:12", + "text": "Thanks, Melanie. Here's one- 'Embracing Identity' is all about finding comfort and love in being yourself. The woman in the painting stands for the journey of acceptance. My aim was to show warmth, love and self-acceptance." + }, + { + "speaker": "Melanie", + "blip_caption": "a photo of a woman is making a vase on a wheel", + "dia_id": "D11:13", + "text": "Wow, Caroline, that's gorgeous! I love the self-acceptance and love theme. How does art help you with your self-discovery and acceptance journey?" + }, + { + "speaker": "Caroline", + "dia_id": "D11:14", + "text": "Art's allowed me to explore my transition and my changing body. It's been a great way to work through stuff I'm going through. I love that it teaches me to accept the beauty of imperfections." + }, + { + "speaker": "Melanie", + "dia_id": "D11:15", + "text": "Wow, Caroline, that's so cool! Art can be so healing and a way to really connect with who you are. It's awesome that beauty can be found in the imperfections. We're all individual and wonderfully imperfect. Thanks for sharing it with me!" + }, + { + "speaker": "Caroline", + "dia_id": "D11:16", + "text": "Thanks, Melanie. It means a lot to share this with you." + }, + { + "speaker": "Melanie", + "dia_id": "D11:17", + "text": "Great chatting with you! Feel free to reach out any time." + } + ], + "session_12_date_time": "1:50 pm on 17 August, 2023", + "session_12": [ + { + "speaker": "Caroline", + "dia_id": "D12:1", + "text": "Hey Mel! How're ya doin'? Recently, I had a not-so-great experience on a hike. I ran into a group of religious conservatives who said something that really upset me. It made me think how much work we still have to do for LGBTQ rights. It's been so helpful to have people around me who accept and support me, so I know I'll be ok!" + }, + { + "speaker": "Melanie", + "dia_id": "D12:2", + "text": "Hey Caroline, sorry about the hike. It sucks when people are so closed-minded. Strong support really helps. FYI, I finished another pottery project - want to see a pic?" + }, + { + "speaker": "Caroline", + "dia_id": "D12:3", + "text": "Sure thing, Melanie! Can't wait to see your pottery project. I'm happy you found something that makes you happy. Show me when you can!" + }, + { + "speaker": "Melanie", + "img_url": [ + "https://omceramic.com/cdn/shop/products/IMG_0022.jpg" + ], + "blip_caption": "a photo of a bowl with a colorful design on it", + "query": "pottery project ceramic bowl", + "dia_id": "D12:4", + "text": "Here it is. Pretty proud of it! It was a great experience. Thoughts?" + }, + { + "speaker": "Caroline", + "dia_id": "D12:5", + "text": "That bowl is awesome, Mel! What gave you the idea for all the colors and patterns?" + }, + { + "speaker": "Melanie", + "dia_id": "D12:6", + "text": "Thanks, Caroline! I'm obsessed with those, so I made something to catch the eye and make people smile. Plus, painting helps me express my feelings and be creative. Each stroke carries a part of me." + }, + { + "speaker": "Caroline", + "dia_id": "D12:7", + "text": "That's amazing! You put so much effort and passion into it. Your creativity really shines. Seeing how art can be a source of self-expression and growth is truly inspiring. You're killing it!" + }, + { + "speaker": "Melanie", + "dia_id": "D12:8", + "text": "Thanks, Caroline! Your words really mean a lot. I've always felt a strong connection to art, and it's been a huge learning experience. It's both a sanctuary and a source of comfort. I'm so glad to have something that brings me so much happiness and fulfillment." + }, + { + "speaker": "Caroline", + "dia_id": "D12:9", + "text": "Glad you found something that makes you so happy! Surrounding ourselves with things that bring us joy is important. Life's too short to do anything else!" + }, + { + "speaker": "Melanie", + "dia_id": "D12:10", + "text": "Agreed, Caroline. Life's tough but it's worth it when we have things that make us happy." + }, + { + "speaker": "Caroline", + "dia_id": "D12:11", + "text": "Definitely, Mel! Finding those happy moments and clinging to them is key. It's what keeps us going, even when life's hard. I'm lucky to have people like you to remind me." + }, + { + "speaker": "Melanie", + "dia_id": "D12:12", + "text": "Yeah, same here Caroline. You make life's struggles more bearable." + }, + { + "speaker": "Caroline", + "dia_id": "D12:13", + "text": "Thanks, Melanie! It means a lot having you in my corner. Appreciate our friendship!" + }, + { + "speaker": "Melanie", + "dia_id": "D12:14", + "text": "I appreciate our friendship too, Caroline. You've always been there for me." + }, + { + "speaker": "Caroline", + "img_url": [ + "https://media2.fdncms.com/portmerc/imager/u/large/46577490/pride2022-2-jankowski.jpg" + ], + "blip_caption": "a photo of a group of people walking down a street with balloons", + "query": "friends pride festival", + "dia_id": "D12:15", + "text": "I'm always here for you, Mel! We had a blast last year at the Pride fest. Those supportive friends definitely make everything worth it!" + }, + { + "speaker": "Melanie", + "dia_id": "D12:16", + "text": "That was a blast! So much fun with the whole gang! Wanna do a family outing this summer?" + }, + { + "speaker": "Caroline", + "dia_id": "D12:17", + "text": "Right, it was so much fun! We could do a family outting, or wanna plan something special for this summer, just us two? It'd be a great chance to catch up and explore nature! What do you think?" + }, + { + "speaker": "Melanie", + "dia_id": "D12:18", + "text": "Sounds great, Caroline! Let's plan something special!" + }, + { + "speaker": "Caroline", + "dia_id": "D12:19", + "text": "Sounds great, Mel! We'll make some awesome memories!" + }, + { + "speaker": "Melanie", + "dia_id": "D12:20", + "text": "Yeah, Caroline! I'll start thinking about what we can do." + }, + { + "speaker": "Caroline", + "dia_id": "D12:21", + "text": "Yeah, Mel! Life's all about creating memories. Can't wait for the trip!" + } + ], + "session_13_date_time": "3:31 pm on 23 August, 2023", + "session_13": [ + { + "speaker": "Caroline", + "img_url": [ + "https://i.redd.it/pyq31v7eh6ra1.jpg" + ], + "blip_caption": "a photo of a sign with a picture of a guinea pig", + "query": "adoption brochures application forms external adoption advice assistance group", + "dia_id": "D13:1", + "text": "Hi Melanie! Hope you're doing good. Guess what I did this week? I took the first step towards becoming a mom - I applied to adoption agencies! It's a big decision, but I think I'm ready to give all my love to a child. I got lots of help from this adoption advice/assistance group I attended. It was great!" + }, + { + "speaker": "Melanie", + "dia_id": "D13:2", + "text": "Caroline, congrats! So proud of you for taking this step. How does it feel? Also, do you have any pets?" + }, + { + "speaker": "Caroline", + "dia_id": "D13:3", + "text": "Thanks, Mel! Exciting but kinda nerve-wracking. Parenting's such a big responsibility. And yup, I do- Oscar, my guinea pig. He's been great. How are your pets?" + }, + { + "speaker": "Melanie", + "img_url": [ + "https://i.redd.it/kgggim1gom951.jpg" + ], + "blip_caption": "a photo of a black dog laying in the grass with a frisbee", + "query": "pets Luna Oliver playing frisbee backyard", + "dia_id": "D13:4", + "text": "Yeah, it's normal to be both excited and nervous with a big decision. And thanks for asking, they're good- we got another cat named Bailey too. Here's a pic of Oliver. Can you show me one of Oscar?" + }, + { + "speaker": "Caroline", + "img_url": [ + "https://cdn.i-scmp.com/sites/default/files/styles/landscape/public/d8/yp/images/shutterstock533807500.jpg" + ], + "blip_caption": "a photography of a guinea in a cage with hay and hay", + "query": "oscar munching parsley playpen", + "dia_id": "D13:5", + "re-download": true, + "text": "He's so cute! What’s the funniest thing Oliver's done? And sure, check out this pic of him eating parsley! Veggies are his fave!" + }, + { + "speaker": "Melanie", + "img_url": [ + "https://i.redd.it/fgv0i3nzo7541.jpg" + ], + "blip_caption": "a photo of a person holding a carrot in front of a horse", + "query": "oscar carrot in mouth", + "dia_id": "D13:6", + "text": "Oliver's hilarious! He hid his bone in my slipper once! Cute, right? Almost as silly as when I got to feed a horse a carrot. " + }, + { + "speaker": "Caroline", + "dia_id": "D13:7", + "text": "That's so funny! I used to go horseback riding with my dad when I was a kid, we'd go through the fields, feeling the wind. It was so special. I've always had a love for horses!" + }, + { + "speaker": "Melanie", + "img_url": [ + "https://warpedtable.com/cdn/shop/products/F331B563-AB73-430A-A6DF-3C5E0F91A4D8.jpg" + ], + "blip_caption": "a photo of a horse painted on a wooden wall", + "query": "horse painting", + "dia_id": "D13:8", + "text": "Wow, that sounds great - I agree, they're awesome. Here's a photo of my horse painting I did recently." + }, + { + "speaker": "Caroline", + "dia_id": "D13:9", + "text": "Wow, Melanie, that's amazing! Love all the details and how you got the horse's grace and strength. Do you like painting animals?" + }, + { + "speaker": "Melanie", + "dia_id": "D13:10", + "text": "Thanks, Caroline! Glad you like it. Yeah, I love to. It's peaceful and special. Horses have such grace! Do you like to paint too?" + }, + { + "speaker": "Caroline", + "img_url": [ + "https://cdn.shopify.com/s/files/1/0302/3968/6755/files/IMG_8385_a145b124-53ab-4b3c-8f1a-497fa2d39a49.jpg" + ], + "blip_caption": "a photo of a painting of a woman with a blue face", + "query": "self-portrait painting vibrant colors", + "dia_id": "D13:11", + "text": "Painting's great for expressing myself. I love creating art! Here's a recent self-portrait I made last week." + }, + { + "speaker": "Melanie", + "dia_id": "D13:12", + "text": "Caroline, that's great! The blue's really powerful, huh? How'd you feel while painting it?" + }, + { + "speaker": "Caroline", + "dia_id": "D13:13", + "text": "Thanks, Mel! I felt liberated and empowered doing it. Painting helps me explore my identity and be true to myself. It's definitely therapeutic." + }, + { + "speaker": "Melanie", + "dia_id": "D13:14", + "text": "Wow, Caroline, that's great! Art's awesome for showing us who we really are and getting in touch with ourselves. What else helps you out?" + }, + { + "speaker": "Caroline", + "dia_id": "D13:15", + "text": "Thanks, Melanie. Art gives me a sense of freedom, but so does having supportive people around, promoting LGBTQ rights and being true to myself. I want to live authentically and help others to do the same." + }, + { + "speaker": "Melanie", + "dia_id": "D13:16", + "text": "Wow, Caroline! That's amazing. You really care about being real and helping others. Wishing you the best on your adoption journey!" + }, + { + "speaker": "Caroline", + "dia_id": "D13:17", + "text": "Thanks, Melanie! I really appreciate it. Excited for the future! Bye!" + }, + { + "speaker": "Melanie", + "dia_id": "D13:18", + "text": "Bye Caroline. I'm here for you. Take care of yourself." + } + ], + "session_14_date_time": "1:33 pm on 25 August, 2023", + "session_14": [ + { + "speaker": "Caroline", + "img_url": [ + "https://photos.thetrek.co/wp-content/uploads/2017/11/IMG_1742-e1509796327550.jpg" + ], + "blip_caption": "a photo of a woman sitting on a sign on top of a mountain", + "query": "letter apology hike encounter", + "dia_id": "D14:1", + "text": "Hey, Mel! How's it going? There's something I want to tell you. I went hiking last week and got into a bad spot with some people. It really bugged me, so I tried to apologize to them." + }, + { + "speaker": "Melanie", + "img_url": [ + "https://i0.wp.com/bardith.com/wp-content/uploads/2022/05/IMG_4371-1.jpg" + ], + "blip_caption": "a photo of a plate with a bunch of flowers on it", + "query": "pottery purple bowl floral patterns", + "dia_id": "D14:2", + "text": "Wow, Caroline! Sorry that happened to you. It's tough when those things happen, but it's great you apologized. Takes a lot of courage and maturity! What do you think of this?" + }, + { + "speaker": "Caroline", + "dia_id": "D14:3", + "text": "Thanks, Melanie! That plate is awesome! Did you make it?" + }, + { + "speaker": "Melanie", + "dia_id": "D14:4", + "text": "Yeah, I made it in pottery class yesterday. I love it! Pottery's so relaxing and creative. Have you tried it yet?" + }, + { + "speaker": "Caroline", + "img_url": [ + "https://i0.wp.com/makesomethingmondays.com/wp-content/uploads/2017/07/mini-beach-sunset-painting-diy.jpg" + ], + "blip_caption": "a photo of a painting of a sunset on a small easel", + "query": "vibrant sunset beach painting", + "dia_id": "D14:5", + "text": "Nah, I haven't. I've been busy painting - here's something I just finished." + }, + { + "speaker": "Melanie", + "dia_id": "D14:6", + "text": "Wow Caroline, that looks amazing! Those colors are so vivid, it really looks like a real sunset. What gave you the idea to paint it?" + }, + { + "speaker": "Caroline", + "blip_caption": "a photo of a painting of a sunset over the ocean", + "dia_id": "D14:7", + "text": "Thanks, Melanie! I painted it after I visited the beach last week. Just seeing the sun dip below the horizon, all the amazing colors - it was amazing and calming. So I just had to try to capture that feeling in my painting." + }, + { + "speaker": "Melanie", + "dia_id": "D14:8", + "text": "Wow, the beach really inspired you. The art really took me to that moment and I can feel the serenity. You captured the sunset perfectly, so peaceful!" + }, + { + "speaker": "Caroline", + "dia_id": "D14:9", + "text": "Thanks Mel, really appreciate your kind words. It means a lot to me that you can feel the sense of peace and serenity. Makes me feel connected." + }, + { + "speaker": "Melanie", + "dia_id": "D14:10", + "text": "I feel the same way! Art is so cool like that - it connects us and helps us understand each other. I was actually just remembering yesterday, spending the day with my fam volunteering at a homeless shelter. It was hard to see how neglected some people are, but it was great to feel like we could make a difference." + }, + { + "speaker": "Caroline", + "img_url": [ + "https://images.squarespace-cdn.com/content/v1/64cda0c3f2719a0e6e707684/a08e6e1f-f0e0-4f1a-b567-1f1f92b80aab/35970846_829192503937065_1026209343625756672_o_829192493937066.jpg" + ], + "blip_caption": "a photo of a crowd of people walking down a street with a rainbow flag", + "query": "volunteering pride event", + "dia_id": "D14:11", + "text": "Wow, Mel, you're amazing! Volunteering and making a difference- it's so heartwarming. You're an inspiration to us all!" + }, + { + "speaker": "Melanie", + "blip_caption": "a photo of a bulletin board with a rainbow flag and a don't ever be afraid to", + "dia_id": "D14:12", + "text": "Thanks, Caroline! I really appreciate your help and motivation. What made you decide to transition and join the transgender community?" + }, + { + "speaker": "Caroline", + "img_url": [ + "https://npr.brightspotcdn.com/legacy/sites/wuwm/files/201811/20181029_095916.jpg" + ], + "blip_caption": "a photo of a building with a large eagle painted on it", + "query": "rainbow flag painting unity acceptance", + "dia_id": "D14:13", + "text": "Finding a community where I'm accepted, loved and supported has really meant a lot to me. It's made a huge difference to have people who get what I'm going through. Stuff like this mural are really special to me!" + }, + { + "speaker": "Melanie", + "dia_id": "D14:14", + "text": "Caroline, glad you found a supportive community! Can you tell me more about why it's special to you?" + }, + { + "speaker": "Caroline", + "blip_caption": "a photo of a stained glass window with a picture of a person on a horse", + "dia_id": "D14:15", + "text": "The rainbow flag mural is important to me as it reflects the courage and strength of the trans community. The eagle symbolizes freedom and pride, representing my own resilience and that of others." + }, + { + "speaker": "Melanie", + "blip_caption": "a photo of a stained glass window with a person holding a key", + "dia_id": "D14:16", + "text": "I'm in awe of your courage as a trans person. Have you made any more art lately?" + }, + { + "speaker": "Caroline", + "img_url": [ + "https://projects.history.qmul.ac.uk/thehistorian/wp-content/uploads/sites/24/2017/10/IMG_20170922_072615_165.jpg" + ], + "blip_caption": "a photo of three stained glass windows in a church with a clock", + "query": "stained glass window letter", + "dia_id": "D14:17", + "text": "Thanks, Mel! I made this stained glass window to remind myself and others that within us all is the key to discovering our true potential and living our best life." + }, + { + "speaker": "Melanie", + "dia_id": "D14:18", + "text": "Wow, Caroline, that looks amazing! What inspired it?" + }, + { + "speaker": "Caroline", + "blip_caption": "a photo of a large stained glass window in a church", + "dia_id": "D14:19", + "text": "Thanks! It was made for a local church and shows time changing our lives. I made it to show my own journey as a transgender woman and how we should accept growth and change." + }, + { + "speaker": "Melanie", + "blip_caption": "a photo of a door with a stained glass window and a coat rack", + "dia_id": "D14:20", + "text": "Wow, Caroline! All those colors are incredible and the story it tells is so inspiring." + }, + { + "speaker": "Caroline", + "img_url": [ + "https://i0.wp.com/marbleheadcurrent.org/wp-content/uploads/2023/07/rainbow.jpg" + ], + "blip_caption": "a photo of a painted sidewalk with a rainbow design on it", + "query": "painting rainbow flag unity acceptance", + "dia_id": "D14:21", + "text": "Thanks, Mel! Glad you like it. It's a symbol of togetherness, to celebrate differences and be that much closer. I'd love to make something like this next!" + }, + { + "speaker": "Melanie", + "dia_id": "D14:22", + "text": "Wow, that's gorgeous! Where did you find it?" + }, + { + "speaker": "Caroline", + "dia_id": "D14:23", + "text": "I was out walking in my neighborhood when I came across this cool rainbow sidewalk for Pride Month. It was so vibrant and welcoming, I had to take a picture! It reminds us that love and acceptance are everywhere—even where we least expect it." + }, + { + "speaker": "Melanie", + "blip_caption": "a photo of a person drawing a flower on the ground", + "dia_id": "D14:24", + "text": "That's so nice, Caroline! Art can be in the most unlikely places. Love and acceptance really can be found everywhere." + }, + { + "speaker": "Caroline", + "img_url": [ + "https://static.skillshare.com/uploads/project/281358/cover_full_592aef91cce2432e71c739804161e0fb.jpg" + ], + "blip_caption": "a photo of a painting of flowers and a watercolor palette", + "query": "drawing flower ground colored chalk simple act creativity", + "dia_id": "D14:25", + "text": "Agreed, Mel! Art can be a real mood-booster - I saw someone drawing on the ground the other day and it made me so happy. Creativity sure can lighten someone's day." + }, + { + "speaker": "Melanie", + "dia_id": "D14:26", + "text": "Wow, Caroline, that's so nice! The colors are so bright and the flowers are so pretty. Art is such a source of joy." + }, + { + "speaker": "Caroline", + "img_url": [ + "https://i.pinimg.com/originals/3d/e3/b8/3de3b8a013be3eec63cc454cb0c63536.jpg" + ], + "blip_caption": "a photo of a drawing of a bunch of flowers on a table", + "query": "bouquet wildflowers art", + "dia_id": "D14:27", + "text": "Thanks, Mel! Art gives me so much joy. It helps me show my feelings and freeze gorgeous moments, like a bouquet of flowers. " + }, + { + "speaker": "Melanie", + "dia_id": "D14:28", + "text": "Wow, did you make that? It looks so real!" + }, + { + "speaker": "Caroline", + "blip_caption": "a photo of a drawing of a flower bouquet with a person holding it", + "dia_id": "D14:29", + "text": "Yeah, definitely! Drawing flowers is one of my faves. Appreciating nature and sharing it is great. What about you, Mel? What type of art do you love?" + }, + { + "speaker": "Melanie", + "img_url": [ + "https://i.redd.it/k9wcp85ledi91.jpg" + ], + "blip_caption": "a photo of a painting of a sunflower on a canvas", + "query": "painting field sunflowers", + "dia_id": "D14:30", + "text": "Painting landscapes and still life is my favorite! Nature's amazing, here's a painting I did recently." + }, + { + "speaker": "Caroline", + "dia_id": "D14:31", + "text": "Wow, Mel! Any more paintings coming up?" + }, + { + "speaker": "Melanie", + "dia_id": "D14:32", + "text": "I'm feeling inspired by autumn so I'm planning a few. You got any cool art projects coming up?" + }, + { + "speaker": "Caroline", + "dia_id": "D14:33", + "text": "I'm putting together an LGBTQ art show next month and I'm gonna show my paintings. Super stoked!" + }, + { + "speaker": "Melanie", + "dia_id": "D14:34", + "text": "Wow, Caroline, that's awesome! Can't wait to see your show - the LGBTQ community needs more platforms like this!" + }, + { + "speaker": "Caroline", + "blip_caption": "a photo of a poster for a concert with a man in a cowboy hat", + "dia_id": "D14:35", + "text": "Yeah Mel, stoked! Gonna be a great night featuring LGBTQ artists and their awesome talents. We want it to spread understanding and acceptance - let's make it happen!" + } + ], + "session_15_date_time": "3:19 pm on 28 August, 2023", + "session_15": [ + { + "speaker": "Caroline", + "dia_id": "D15:1", + "text": "Hey Melanie, great to hear from you. What's been up since we talked?" + }, + { + "speaker": "Melanie", + "img_url": [ + "https://img-aws.ehowcdn.com/1280x/www.onlyinyourstate.com/wp-content/uploads/2022/12/gym8.jpg" + ], + "blip_caption": "a photo of a playground with a climbing net and a slide", + "query": "kids climbing jungle gym park", + "dia_id": "D15:2", + "text": "Hey Caroline! Since we last spoke, I took my kids to a park yesterday. They had fun exploring and playing. It was nice seeing them have a good time outdoors. Time flies, huh? What's new with you?" + }, + { + "speaker": "Caroline", + "dia_id": "D15:3", + "text": "Wow, your kids had so much fun at the park! Being outdoors can be really enjoyable. A lot happened since our last chat. I've been chasing my ambitions and had the chance to volunteer at an LGBTQ+ youth center. It was so gratifying to talk to similar young people. It made me remember how essential it is to be kind and show support." + }, + { + "speaker": "Melanie", + "dia_id": "D15:4", + "text": "That sounds great, Caroline. Volunteering is a great way to meet people. Creating community and supporting each other, especially for kids, is really important. How did you feel about your time there? Anything that sticks out to you?" + }, + { + "speaker": "Caroline", + "blip_caption": "a photo of a table with a black table cloth and a group of people", + "dia_id": "D15:5", + "text": "I loved it. It was awesome to see how strong the young people were, with all the challenges they face. I felt fulfilled guiding and supporting them. I even got to let them know they're not alone by sharing my story. Such a powerful, emotional experience." + }, + { + "speaker": "Melanie", + "dia_id": "D15:6", + "text": "Was connecting with those young folks meaningful for you? " + }, + { + "speaker": "Caroline", + "dia_id": "D15:7", + "text": "It was so special to me. It reminded me of my own struggles in the past and how I felt alone. I was glad I could share my story and offer them support - it felt like I could make a difference." + }, + { + "speaker": "Melanie", + "dia_id": "D15:8", + "text": "That's great. Sharing your story and support might make a difference for a long time. What do you hope to do next time?" + }, + { + "speaker": "Caroline", + "dia_id": "D15:9", + "text": "I'm definitely carrying on volunteering at the youth center. It's an important part of my life and I've made strong connections with people there. I really believe in community and supporting each other. So I wanna keep making a difference." + }, + { + "speaker": "Melanie", + "dia_id": "D15:10", + "text": "That's great news, Caroline! Love seeing your dedication to helping others. Any specific projects or activities you're looking forward to there?" + }, + { + "speaker": "Caroline", + "dia_id": "D15:11", + "text": "We're putting together a talent show for the kids next month. I'm looking forward to seeing how much fun everyone has and how proud they'll feel of their talents!" + }, + { + "speaker": "Melanie", + "img_url": [ + "https://www.stomplight.com/cdn/shop/products/DavidAguilar.jpg" + ], + "blip_caption": "a photo of a band playing on a stage in a park", + "query": "talent show stage colorful lights microphone", + "dia_id": "D15:12", + "text": "That's so cool, Caroline! That's a great way to show off and be proud of everyone's skills. You know I love live music. Can't wait to hear about it!" + }, + { + "speaker": "Caroline", + "dia_id": "D15:13", + "text": "Wow! Did you see that band?" + }, + { + "speaker": "Melanie", + "blip_caption": "a photo of a crowd of people at a concert with their hands in the air", + "dia_id": "D15:14", + "text": "Yeah, that pic was from a show I went to. It was so much fun and reminded me of how music brings us together." + }, + { + "speaker": "Caroline", + "dia_id": "D15:15", + "text": "Wow, what a fun moment! What's the band?" + }, + { + "speaker": "Melanie", + "dia_id": "D15:16", + "text": "\"Summer Sounds\"- The playing an awesome pop song that got everyone dancing and singing. It was so fun and lively!" + }, + { + "speaker": "Caroline", + "blip_caption": "a photo of a man playing a guitar in a recording studio", + "dia_id": "D15:17", + "text": "That sounds great! Music brings us together and brings joy. Playing and singing let me express myself and connect with others - love it! So cathartic and uplifting." + }, + { + "speaker": "Melanie", + "dia_id": "D15:18", + "text": "Cool! What type of music do you play?" + }, + { + "speaker": "Caroline", + "blip_caption": "a photo of a guitar on display in a store", + "dia_id": "D15:19", + "text": "Guitar's mostly my thing. Playing it helps me get my emotions out." + }, + { + "speaker": "Melanie", + "dia_id": "D15:20", + "text": "That's awesome! What type of guitar? Been playing long?" + }, + { + "speaker": "Caroline", + "dia_id": "D15:21", + "text": "I started playing acoustic guitar about five years ago; it's been a great way to express myself and escape into my emotions." + }, + { + "speaker": "Melanie", + "dia_id": "D15:22", + "text": "Music's amazing, isn't it? Any songs that have deep meaning for you?" + }, + { + "speaker": "Caroline", + "dia_id": "D15:23", + "text": "Yeah totally! \"Brave\" by Sara Bareilles has a lot of significance for me. It's about being courageous and fighting for what's right. Whenever I hear this jam, I think about the paths I've taken and the progress I've made." + }, + { + "speaker": "Melanie", + "blip_caption": "a photo of a piece of paper with a drawing of a man playing a piano", + "dia_id": "D15:24", + "text": "That's a gorgeous song, Caroline. It really fits with your journey and your determination to make a difference. Music can be so inspiring and uplifting." + }, + { + "speaker": "Caroline", + "dia_id": "D15:25", + "text": "Thanks, Melanie! Appreciate it. You play any instruments?" + }, + { + "speaker": "Melanie", + "blip_caption": "a photo of a sheet music with notes and a pencil", + "dia_id": "D15:26", + "text": "Yeah, I play clarinet! Started when I was young and it's been great. Expression of myself and a way to relax." + }, + { + "speaker": "Caroline", + "dia_id": "D15:27", + "text": "Cool! Got any fav tunes?" + }, + { + "speaker": "Melanie", + "blip_caption": "a photo of a laptop computer with a graph on it", + "dia_id": "D15:28", + "text": "I'm a fan of both classical like Bach and Mozart, as well as modern music like Ed Sheeran's \"Perfect\"." + } + ], + "session_16_date_time": "12:09 am on 13 September, 2023", + "session_16": [ + { + "speaker": "Caroline", + "img_url": [ + "https://assets.simpleviewinc.com/simpleview/image/upload/c_fill,f_jpg,h_371,q_75,w_640/v1/crm/corpuschristitx/Sunset-Lake-Park_38118D81-5056-A36F-23E62D2F41525FF4-38118cb45056a36_381194bb-5056-a36f-23c599c63a3950d7.jpg" + ], + "blip_caption": "a photo of a beach with a fence and a sunset", + "query": "sunset lake", + "dia_id": "D16:1", + "text": "Hey Mel, long time no chat! I had a wicked day out with the gang last weekend - we went biking and saw some pretty cool stuff. It was so refreshing, and the pic I'm sending is just stunning, eh?" + }, + { + "speaker": "Melanie", + "img_url": [ + "https://exploringtheprime.com/wp-content/uploads/2019/10/IMG_6705-2.jpg" + ], + "blip_caption": "a photo of a dirt road surrounded by trees with yellow leaves", + "query": "family hiking trail vibrant autumn colors", + "dia_id": "D16:2", + "text": "Hey Caroline! It's so good to hear from you! That pic is so beautiful, the colors really pop. Biking sounds like a great way to get out in nature. We went camping with the kids a few weeks ago, had a blast exploring the forest and hiking. Nature can be so refreshing for your soul. Any plans coming up?" + }, + { + "speaker": "Caroline", + "dia_id": "D16:3", + "text": "Melanie, that photo's amazing! I love all the yellow leaves, it looks so cozy. That sounds like fun! Seeing how excited they get for the little things is awesome, it's so contagious." + }, + { + "speaker": "Melanie", + "dia_id": "D16:4", + "text": "Thanks, Caroline! It's awesome seeing the kids get excited learning something new about nature. Those moments make being a parent worth it. We roasted marshmallows and shared stories around the campfire. Those simple moments make the best memories. What inspires you with your volunteering?" + }, + { + "speaker": "Caroline", + "img_url": [ + "https://i.pinimg.com/originals/34/2e/72/342e72b194865e01a38af86c307e95c7.jpg" + ], + "blip_caption": "a photo of a painting of a heart on a table", + "query": "canvas painting rainbow colors", + "dia_id": "D16:5", + "text": "I'm inspired seeing my work make a difference for the LGBTQ+ community. Knowing I'm helping create a more loving world is amazing. I'm really thankful for my friends, family and mentors' support. It inspires me to keep making art, too." + }, + { + "speaker": "Melanie", + "dia_id": "D16:6", + "text": "Wow, Caroline, that looks awesome! I love how it shows the togetherness and power you were talking about. How long have you been creating art?" + }, + { + "speaker": "Caroline", + "dia_id": "D16:7", + "text": "Since I was 17 or so. I find it soempowering and cathartic. It's amazing how art can show things that are hard to put into words. How long have you been into art?" + }, + { + "speaker": "Melanie", + "img_url": [ + "https://www.1hotpieceofglass.com/cdn/shop/files/image_93ad5985-ff65-4b93-877b-3ee948ac5641_5000x.jpg" + ], + "blip_caption": "a photo of a group of bowls and a starfish on a white surface", + "query": "pottery bowl intricate patterns purple glaze", + "dia_id": "D16:8", + "text": "Seven years now, and I've finally found my real muses: painting and pottery. It's so calming and satisfying. Check out my pottery creation in the pic!" + }, + { + "speaker": "Caroline", + "dia_id": "D16:9", + "text": "Melanie, those bowls are amazing! They each have such cool designs. I love that you chose pottery for your art. Painting and drawing have helped me express my feelings and explore my gender identity. Creating art was really important to me during my transition - it helped me understand and accept myself. I'm so grateful." + }, + { + "speaker": "Melanie", + "dia_id": "D16:10", + "text": "Thanks, Caroline! It has really helped me out. I love how it's both a creative outlet and a form of therapy. Have you ever thought about trying it or another art form?" + }, + { + "speaker": "Caroline", + "img_url": [ + "https://i.redd.it/z8zsh53ycfvb1.jpg" + ], + "blip_caption": "a photo of a painting on a easel with a red and blue background", + "query": "canvas colourful brush strokes", + "dia_id": "D16:11", + "text": "I haven't done pottery yet, but I'm game for trying new art. I might try it sometime! Check out this piece I made!" + }, + { + "speaker": "Melanie", + "dia_id": "D16:12", + "text": "Wow, Caroline! This painting is awesome. Love the red and blue. What gave you the idea?" + }, + { + "speaker": "Caroline", + "dia_id": "D16:13", + "text": "Thanks, Melanie! I made this painting to show my path as a trans woman. The red and blue are for the binary gender system, and the mix of colors means smashing that rigid thinking. It's a reminder to love my authentic self - it's taken a while to get here but I'm finally proud of who I am." + }, + { + "speaker": "Melanie", + "dia_id": "D16:14", + "text": "Wow, Caro, that painting is amazing! You've made so much progress. I'm super proud of you for being your true self. What effect has the journey had on your relationships?" + }, + { + "speaker": "Caroline", + "dia_id": "D16:15", + "text": "Thanks, Melanie. It's definitely changed them. Some close friends kept supporting me, but a few weren't able to handle it. It wasn't easy, but I'm much happier being around those who accept and love me. Now my relationships feel more genuine." + }, + { + "speaker": "Melanie", + "img_url": [ + "https://i.redd.it/epuj1xq8eaga1.jpg" + ], + "blip_caption": "a photo of a sign posted on a door stating that someone is not being able to leave", + "query": "me kids park joy love happiness", + "dia_id": "D16:16", + "text": "Caroline, it's got to be tough dealing with those changes. Glad you've found people who uplift and accept you! Here's to a good time at the café last weekend - they even had thoughtful signs like this! It brings me so much happiness." + }, + { + "speaker": "Caroline", + "dia_id": "D16:17", + "text": "Whoa, Mel, that sign looks serious. Did anything happen?" + }, + { + "speaker": "Melanie", + "dia_id": "D16:18", + "text": "The sign was just a precaution, I had a great time. But thank you for your concern, you're so thoughtful!" + }, + { + "speaker": "Caroline", + "dia_id": "D16:19", + "text": "Phew! Glad it all worked out and you had a good time at the park!" + }, + { + "speaker": "Melanie", + "dia_id": "D16:20", + "text": "Yeah, it was so much fun! Those joyful moments definitely show us life's beauty." + } + ], + "session_17_date_time": "10:31 am on 13 October, 2023", + "session_17": [ + { + "speaker": "Caroline", + "dia_id": "D17:1", + "text": "Hey Mel, what's up? Long time no see! I just contacted my mentor for adoption advice. I'm ready to be a mom and share my love and family. It's a great feeling. Anything new with you? Anything exciting going on?" + }, + { + "speaker": "Melanie", + "dia_id": "D17:2", + "text": "Hey Caroline! Great to hear from you! Wow, what an amazing journey. Congrats!" + }, + { + "speaker": "Caroline", + "dia_id": "D17:3", + "text": "Thanks, Melanie! I'm stoked to start this new chapter. It's been a dream to adopt and provide a safe, loving home for kids who need it. Do you have any experience with adoption, or know anyone who's gone through the process?" + }, + { + "speaker": "Melanie", + "dia_id": "D17:4", + "text": "Yeah, a buddy of mine adopted last year. It was a long process, but now they're super happy with their new kid. Makes me feel like maybe I should do it too!" + }, + { + "speaker": "Caroline", + "dia_id": "D17:5", + "text": "That's great news about your friend! It can be tough, but so worth it. It's a great way to add to your family and show your love. If you ever do it, let me know — I'd love to help in any way I can." + }, + { + "speaker": "Melanie", + "dia_id": "D17:6", + "text": "Thanks, Caroline! Appreciate your help. Got any tips for getting started on it?" + }, + { + "speaker": "Caroline", + "dia_id": "D17:7", + "text": "Yep! Do your research and find an adoption agency or lawyer. They'll help with the process and provide all the info. Gather documents like references, financial info and medical checks. Don't forget to prepare emotionally, since the wait can be hard. It's all worth it in the end though." + }, + { + "speaker": "Melanie", + "dia_id": "D17:8", + "text": "Thanks for the tip, Caroline. Doing research and readying myself emotionally makes sense. I'll do that. BTW, recently I had a setback. Last month I got hurt and had to take a break from pottery, which I use for self-expression and peace." + }, + { + "speaker": "Caroline", + "dia_id": "D17:9", + "text": "Oh man, sorry to hear that, Melanie. I hope you're okay. Pottery's a great way to relax, so it must have been tough taking a break. Need any help?" + }, + { + "speaker": "Melanie", + "dia_id": "D17:10", + "text": "Thanks, Caroline. It was tough, but I'm doing ok. Been reading that book you recommended a while ago and painting to keep busy." + }, + { + "speaker": "Caroline", + "dia_id": "D17:11", + "text": "Cool that you have creative outlets. Got any paintings to show? I'd love to check them out." + }, + { + "speaker": "Melanie", + "img_url": [ + "https://trendgallery.art/cdn/shop/files/IMG_2355.jpg" + ], + "blip_caption": "a photo of a painting of a sunset with a pink sky", + "query": "landscape painting vibrant purple sunset autumn", + "dia_id": "D17:12", + "text": "Yeah, Here's one I did last week. It's inspired by the sunsets. The colors make me feel calm. What have you been up to lately, artistically?" + }, + { + "speaker": "Caroline", + "dia_id": "D17:13", + "text": "Wow Mel, that's stunning! Love the colors and the chilled-out sunset vibe. What made you paint it? I've been trying out abstract stuff recently. It's kinda freeing, just putting my feelings on the canvas without too much of a plan. It's like a cool form of self-expression." + }, + { + "speaker": "Melanie", + "img_url": [ + "https://theartwerks.com/cdn/shop/products/image_4c8aee8a-5395-4037-a1d4-f6db3a3b0302.jpg" + ], + "blip_caption": "a photo of a painting on a wall with a blue background", + "query": "abstract painting vibrant colors", + "dia_id": "D17:14", + "text": "Thanks, Caroline! I painted it because it was calming. I've done an abstract painting too, take a look! I love how art lets us get our emotions out." + }, + { + "speaker": "Caroline", + "dia_id": "D17:15", + "text": "Wow, that looks great! The blue adds so much to it. What feelings were you hoping to portray?" + }, + { + "speaker": "Melanie", + "dia_id": "D17:16", + "text": "I wanted a peaceful blue streaks to show tranquility. Blue calms me, so I wanted the painting to have a serene vibe while still having lots of vibrant colors." + }, + { + "speaker": "Caroline", + "blip_caption": "a photo of a poster on a wall in a classroom", + "dia_id": "D17:17", + "text": "Yeah, it's very calming. It's awesome how art can show emotions. By the way, I went to a poetry reading last Fri - it was really powerful! Ever been to one?" + }, + { + "speaker": "Melanie", + "dia_id": "D17:18", + "text": "Nope, never been to something like that. What was it about? What made it so special?" + }, + { + "speaker": "Caroline", + "img_url": [ + "https://hips.hearstapps.com/hmg-prod/images/gettyimages-1675780954.jpg" + ], + "blip_caption": "a photography of a sign that says trans lives matter", + "query": "transgender poetry reading trans pride flags", + "dia_id": "D17:19", + "re-download": true, + "text": "It was a transgender poetry reading where transgender people shared their stories through poetry. It was extra special 'cause it was a safe place for self-expression and it was really empowering to hear others share and celebrate their identities." + }, + { + "speaker": "Melanie", + "dia_id": "D17:20", + "text": "Wow, sounds amazing! What was the event like? Those posters are great!" + }, + { + "speaker": "Caroline", + "img_url": [ + "https://i.redd.it/50qvgfuva33b1.jpg" + ], + "blip_caption": "a photo of a drawing of a woman in a dress", + "query": "transgender flag drawing", + "dia_id": "D17:21", + "text": "The room was electric with energy and support! The posters were amazing, so much pride and strength! It inspired me to make some art." + }, + { + "speaker": "Melanie", + "dia_id": "D17:22", + "text": "That's awesome, Caroline! You drew it? What does it mean to you?" + }, + { + "speaker": "Caroline", + "dia_id": "D17:23", + "text": "Thanks, Melanie! Yeah, I drew it. It stands for freedom and being real. It's like a nudge to always stay true to myself and embrace my womanhood." + }, + { + "speaker": "Melanie", + "dia_id": "D17:24", + "text": "I love it. Showing off our true selves is the best thing ever." + }, + { + "speaker": "Caroline", + "dia_id": "D17:25", + "text": "Yep, Melanie! Being ourselves is such a great feeling. It's an ongoing adventure of learning and growing." + }, + { + "speaker": "Melanie", + "dia_id": "D17:26", + "text": "Yep, Caroline. Life's about learning and exploring. Glad we can be on this trip together." + } + ], + "session_18_date_time": "6:55 pm on 20 October, 2023", + "session_18": [ + { + "speaker": "Melanie", + "img_url": [ + "https://i.redd.it/dl8dki2hm3k81.jpg" + ], + "blip_caption": "a photo of a car dashboard with a white cloth and a steering wheel", + "query": "car accident damaged car airbags deployed roadtrip", + "dia_id": "D18:1", + "text": "Hey Caroline, that roadtrip this past weekend was insane! We were all freaked when my son got into an accident. We were so lucky he was okay. It was a real scary experience. Thankfully it's over now. What's been up since we last talked?" + }, + { + "speaker": "Caroline", + "dia_id": "D18:2", + "text": "Oops, sorry 'bout the accident! Must have been traumatizing for you guys. Thank goodness your son's okay. Life sure can be a roller coaster." + }, + { + "speaker": "Melanie", + "dia_id": "D18:3", + "text": "Yeah, our trip got off to a bad start. I was really scared when we got into the accident. Thankfully, my son's ok and that was a reminder that life is precious and to cherish our family." + }, + { + "speaker": "Caroline", + "dia_id": "D18:4", + "text": "Glad your son is okay, Melanie. Life's unpredictable, but moments like these remind us how important our loved ones are. Family's everything." + }, + { + "speaker": "Melanie", + "img_url": [ + "https://familyadventuresva.files.wordpress.com/2022/03/img_5030.jpg" + ], + "blip_caption": "a photo of two children standing on a rocky cliff overlooking a canyon", + "query": "grand canyon family photo", + "dia_id": "D18:5", + "text": "Yeah, you're right, Caroline. Family's super important to me. Especially after the accident, I've thought a lot about how much I need them. They mean the world to me and I'm so thankful to have them. Thankfully, they enjoyed the Grand Canyon a lot!" + }, + { + "speaker": "Caroline", + "dia_id": "D18:6", + "text": "The kids look so cute, Mel! I bet they bring lots of joy. How did they handle the accident?" + }, + { + "speaker": "Melanie", + "dia_id": "D18:7", + "text": "Thanks! They were scared but we reassured them and explained their brother would be OK. They're tough kids." + }, + { + "speaker": "Caroline", + "dia_id": "D18:8", + "text": "Kids are amazingly resilient in tough situations. They have an amazing ability to bounce back." + }, + { + "speaker": "Melanie", + "dia_id": "D18:9", + "text": "They're really amazing. Wish I was that resilient too. But they give me the strength to keep going." + }, + { + "speaker": "Caroline", + "dia_id": "D18:10", + "text": "Our loved ones give us strength to tackle any challenge - it's amazing!" + }, + { + "speaker": "Melanie", + "dia_id": "D18:11", + "text": "Yeah, Caroline. Totally agree. They're my biggest motivation and support." + }, + { + "speaker": "Caroline", + "dia_id": "D18:12", + "text": "It's so sweet to see your love for your family, Melanie. They really are your rock." + }, + { + "speaker": "Melanie", + "dia_id": "D18:13", + "text": "Thanks, Caroline. They're a real support. Appreciate them a lot." + }, + { + "speaker": "Caroline", + "dia_id": "D18:14", + "text": "Glad you've got people to lean on, Melanie. It helps during tougher times." + }, + { + "speaker": "Melanie", + "img_url": [ + "https://live.staticflickr.com/8358/29211988243_82023c5524_b.jpg" + ], + "blip_caption": "a photography of a woman and a child walking on a trail", + "query": "family hiking mountains", + "dia_id": "D18:15", + "re-download": true, + "text": "Yeah for sure. Having my fam around helps a lot. It makes hard times easier." + }, + { + "speaker": "Caroline", + "dia_id": "D18:16", + "text": "Wow, great pic! Is that recent? Looks like you all had fun!" + }, + { + "speaker": "Melanie", + "dia_id": "D18:17", + "text": "Thanks, Caroline! Yup, we just did it yesterday! The kids loved it and it was a nice way to relax after the road trip." + }, + { + "speaker": "Caroline", + "dia_id": "D18:18", + "text": "Glad you got some R&R after the drive. Nature sure seems to refresh us, huh?" + }, + { + "speaker": "Melanie", + "blip_caption": "a photo of a sunset over a body of water", + "dia_id": "D18:19", + "text": "Absolutely! It really helps me reset and recharge. I love camping trips with my fam, 'cause nature brings such peace and serenity." + }, + { + "speaker": "Caroline", + "dia_id": "D18:20", + "text": "Wow, that's awesome! What do you love most about camping with your fam?" + }, + { + "speaker": "Melanie", + "dia_id": "D18:21", + "text": "It's a chance to be present and together. We bond over stories, campfires and nature. It's so peaceful waking up to the sound of birds and the smell of fresh air - it always refreshes my soul." + }, + { + "speaker": "Caroline", + "dia_id": "D18:22", + "text": "That's so peaceful and calming, Melanie! I can picture waking up to nature. It's great that you get to spend quality, tranquil time with your family." + }, + { + "speaker": "Melanie", + "dia_id": "D18:23", + "text": "Thanks, Caroline! This is a great time. Nature and quality time, can't beat it!" + }, + { + "speaker": "Caroline", + "dia_id": "D18:24", + "text": "Yeah totally! They're priceless. Lucky you!" + } + ], + "session_19_date_time": "9:55 am on 22 October, 2023", + "session_19": [ + { + "speaker": "Caroline", + "dia_id": "D19:1", + "text": "Woohoo Melanie! I passed the adoption agency interviews last Friday! I'm so excited and thankful. This is a big move towards my goal of having a family." + }, + { + "speaker": "Melanie", + "img_url": [ + "https://imgur.com/oGlhL5J.jpg" + ], + "blip_caption": "a photo of a couple of wooden dolls sitting on top of a table", + "query": "painted ceramic family figurine", + "dia_id": "D19:2", + "text": "Congrats, Caroline! Adoption sounds awesome. I'm so happy for you. These figurines I bought yesterday remind me of family love. Tell me, what's your vision for the future?" + }, + { + "speaker": "Caroline", + "dia_id": "D19:3", + "text": "Thanks so much, Melanie! It's beautiful! It really brings home how much love's in families - both blood and the ones we choose. I hope to build my own family and put a roof over kids who haven't had that before. For me, adoption is a way of giving back and showing love and acceptance." + }, + { + "speaker": "Melanie", + "dia_id": "D19:4", + "text": "Wow, Caroline, that's awesome. Giving a home to needy kids is such a loving way to build a family. Those kids will be so supported and happy in their new home." + }, + { + "speaker": "Caroline", + "dia_id": "D19:5", + "text": "Thanks, Melanie. My dream is to create a safe and loving home for these kids. Love and acceptance should be everyone's right, and I want them to experience it." + }, + { + "speaker": "Melanie", + "dia_id": "D19:6", + "text": "I totally agree, Caroline. Everyone deserves that. It's awesome to see how passionate you are about helping these kids." + }, + { + "speaker": "Caroline", + "dia_id": "D19:7", + "text": "Thanks, Mel. Finding self-acceptance was a long process, but now I'm ready to offer love and support to those who need it. It's empowering to make a positive difference in someone's life." + }, + { + "speaker": "Melanie", + "dia_id": "D19:8", + "text": "That must have been tough for you, Caroline. Respect for finding acceptance and helping others with what you've been through. You're so strong and inspiring." + }, + { + "speaker": "Caroline", + "dia_id": "D19:9", + "text": "Thanks, Melanie. Transitioning wasn't easy and acceptance wasn't either, but the help I got from friends, family and people I looked up to was invaluable. They boosted me through tough times and helped me find out who I really am. That's why I want to pass that same support to anyone who needs it. Bringing others comfort and helping them grow brings me such joy." + }, + { + "speaker": "Melanie", + "dia_id": "D19:10", + "text": "I'm so happy for you, Caroline. You found your true self and now you're helping others. You're so inspiring!" + }, + { + "speaker": "Caroline", + "blip_caption": "a photo of a clock with a green and yellow design on it", + "dia_id": "D19:11", + "text": "Thanks, Melanie. Your support really means a lot. This journey has been amazing and I'm grateful I get to share it and help others with theirs. It's a real gift." + }, + { + "speaker": "Melanie", + "dia_id": "D19:12", + "text": "Absolutely! I'm so glad we can always be there for each other." + }, + { + "speaker": "Caroline", + "dia_id": "D19:13", + "text": "Glad you agree, Caroline. Appreciate the support of those close to me. Their encouragement made me who I am." + }, + { + "speaker": "Melanie", + "dia_id": "D19:14", + "text": "Glad you had support. Being yourself is great!" + }, + { + "speaker": "Caroline", + "img_url": [ + "https://trendgallery.art/cdn/shop/products/IMG_4482.jpg" + ], + "blip_caption": "a photo of a painting with the words happiness painted on it", + "query": "painting vibrant colors happiness self-expression", + "dia_id": "D19:15", + "text": "Yeah, that's true! It's so freeing to just be yourself and live honestly. We can really accept who we are and be content." + } + ], + "session_20_date_time": "4:10 pm on 26 October, 2023", + "session_21_date_time": "9:35 am on 31 October, 2023", + "session_22_date_time": "12:28 am on 8 November, 2023", + "session_23_date_time": "5:15 pm on 11 November, 2023", + "session_24_date_time": "2:46 pm on 16 November, 2023", + "session_25_date_time": "1:18 pm on 21 November, 2023", + "session_26_date_time": "4:39 pm on 24 November, 2023", + "session_27_date_time": "6:25 pm on 26 November, 2023", + "session_28_date_time": "8:52 pm on 5 December, 2023", + "session_29_date_time": "12:20 am on 8 December, 2023", + "session_30_date_time": "4:37 pm on 10 December, 2023", + "session_31_date_time": "3:24 pm on 16 December, 2023", + "session_32_date_time": "3:43 pm on 20 December, 2023", + "session_33_date_time": "8:32 pm on 27 December, 2023", + "session_34_date_time": "1:08 pm on 30 December, 2023", + "session_35_date_time": "12:19 am on 4 January, 2024" + }, + "qa": [ + { + "question": "Would Caroline pursue writing as a career option?", + "answer": "LIkely no; though she likes reading, she wants to be a counselor", + "evidence": [ + "D7:5", + "D7:9" + ], + "category": 3 + }, + { + "question": "Would Caroline likely have Dr. Seuss books on her bookshelf?", + "answer": "Yes, since she collects classic children's books", + "evidence": [ + "D6:9" + ], + "category": 3 + }, + { + "question": "When did Caroline go to the LGBTQ support group?", + "answer": "7 May 2023", + "evidence": [ + "D1:3" + ], + "category": 2 + }, + { + "question": "When did Caroline go biking with friends?", + "answer": "The weekend before 13 September 2023", + "evidence": [ + "D16:1" + ], + "category": 2 + }, + { + "question": "Who supports Caroline when she has a negative experience?", + "answer": "Her mentors, family, and friends", + "evidence": [ + "D12:1", + "D3:11" + ], + "category": 1 + }, + { + "question": "What activities does Melanie partake in?", + "answer": "pottery, camping, painting, swimming", + "evidence": [ + "D5:4", + "D9:1", + "D1:12", + "D1:18" + ], + "category": 1 + }, + { + "question": "What items has Melanie bought?", + "answer": "Figurines, shoes", + "evidence": [ + "D19:2", + "D7:18" + ], + "category": 1 + }, + { + "question": "Would Caroline want to move back to her home country soon?", + "answer": "No; she's in the process of adopting children.", + "evidence": [ + "D19:1", + "D19:3" + ], + "category": 3 + }, + { + "question": "Would Melanie be considered an ally to the transgender community?", + "answer": "Yes, she is supportive", + "evidence": [], + "category": 3 + }, + { + "question": "When did Melanie go to the museum?", + "answer": "5 July 2023", + "evidence": [ + "D6:4" + ], + "category": 2 + }, + { + "question": "What types of pottery have Melanie and her kids made?", + "answer": "bowls, cup", + "evidence": [ + "D12:14", + "D8:4", + "D5:6" + ], + "category": 1 + }, + { + "question": "What career path has Caroline decided to persue?", + "answer": "counseling or mental health for Transgender people", + "evidence": [ + "D4:13", + "D1:11" + ], + "category": 1 + }, + { + "question": "When did Caroline give a speech at a school?", + "answer": "The week before 9 June 2023", + "evidence": [ + "D3:1" + ], + "category": 2 + }, + { + "question": "Which classical musicians does Melanie enjoy listening to?", + "answer": "Bach and Mozart", + "evidence": [ + "D15:28" + ], + "category": 4 + }, + { + "question": "What subject have Caroline and Melanie both painted?", + "answer": "Sunsets", + "evidence": [ + "D14:5", + "D8:6" + ], + "category": 1 + }, + { + "question": "Would Melanie likely enjoy the song \"The Four Seasons\" by Vivaldi?", + "answer": "Yes; it's classical music", + "evidence": [ + "D15:28" + ], + "category": 3 + }, + { + "question": "When is Caroline's youth center putting on a talent show?", + "answer": "September 2023", + "evidence": [ + "D15:11" + ], + "category": 2 + }, + { + "question": "When did Caroline join a new activist group?", + "answer": "The Tuesday before 20 July 2023", + "evidence": [ + "D10:3" + ], + "category": 2 + }, + { + "question": "What did the posters at the poetry reading say?", + "answer": "\"Trans Lives Matter\"", + "evidence": [ + "D17:19" + ], + "category": 4 + }, + { + "question": "Would Melanie be more interested in going to a national park or a theme park?", + "answer": "National park; she likes the outdoors", + "evidence": [ + "D10:12", + "D10:14" + ], + "category": 3 + }, + { + "question": "Where did Oliver hide his bone once?", + "answer": "In Melanie's slipper", + "evidence": [ + "D13:6" + ], + "category": 4 + }, + { + "question": "What would Caroline's political leaning likely be?", + "answer": "Liberal", + "evidence": [ + "D12:1" + ], + "category": 3 + }, + { + "question": "When did Melanie's family go on a roadtrip?", + "answer": "The weekend before 20 October 2023", + "evidence": [ + "D18:1" + ], + "category": 2 + }, + { + "question": "How many children does Melanie have?", + "answer": 3, + "evidence": [ + "D18:1", + "D18:7" + ], + "category": 1 + }, + { + "question": "Would Melanie go on another roadtrip soon?", + "answer": "Likely no; since this one went badly", + "evidence": [ + "D18:3", + "D18:1" + ], + "category": 3 + }, + { + "question": "When did Melanie go on a hike after the roadtrip?", + "answer": "19 October 2023", + "evidence": [ + "D18:17" + ], + "category": 1 + }, + { + "question": "What is Melanie's hand-painted bowl a reminder of?", + "answer": "art and self-expression", + "evidence": [ + "D4:5" + ], + "category": 4 + }, + { + "question": "When did Melanie go camping in July?", + "answer": "two weekends before 17 July 2023", + "evidence": [ + "D9:1" + ], + "category": 2 + }, + { + "question": "What was discussed in the LGBTQ+ counseling workshop?", + "answer": "therapeutic methods and how to best work with trans people", + "evidence": [ + "D4:13" + ], + "category": 4 + }, + { + "question": "What did Mel and her kids make during the pottery workshop?", + "answer": "pots", + "evidence": [ + "D8:2" + ], + "category": 4 + }, + { + "question": "What personality traits might Melanie say Caroline has?", + "answer": "Thoughtful, authentic, driven", + "evidence": [ + "D16:18", + "D13:16", + "D7:4" + ], + "category": 3 + }, + { + "question": "Would Caroline be considered religious?", + "answer": "Somewhat, but not extremely religious", + "evidence": [ + "D14:19", + "D12:1" + ], + "category": 3 + }, + { + "question": "How did Melanie feel after the accident?", + "answer": "Grateful and thankful for her family", + "evidence": [ + "D18:5" + ], + "category": 4 + }, + { + "question": "What happened to Melanie's son on their road trip?", + "answer": "He got into an accident", + "evidence": [ + "D18:1" + ], + "category": 4 + }, + { + "question": "Would Melanie be considered a member of the LGBTQ community?", + "answer": "Likely no, she does not refer to herself as part of it", + "evidence": [], + "category": 3 + }, + { + "question": "When did Caroline go to a pride parade during the summer?", + "answer": "The week before 3 July 2023", + "evidence": [ + "D5:1" + ], + "category": 2 + }, + { + "question": "In what ways is Caroline participating in the LGBTQ community?", + "answer": "Joining activist group, going to pride parades, participating in an art show, mentoring program", + "evidence": [ + "D10:3", + "D5:1", + "D9:12", + "D9:2" + ], + "category": 1 + } + ] + }, + { + "sample_id": "conv-42", + "conversation": { + "speaker_a": "Joanna", + "speaker_b": "Nate", + "session_1_date_time": "7:31 pm on 21 January, 2022", + "session_1": [ + { + "speaker": "Nate", + "dia_id": "D1:1", + "text": "Hey Joanna! Long time no see! What's up? Anything fun going on?" + }, + { + "speaker": "Joanna", + "dia_id": "D1:2", + "text": "Hey Nate! Long time no see! I've been working on a project lately - it's been pretty cool. What about you - any fun projects or hobbies?" + }, + { + "speaker": "Nate", + "dia_id": "D1:3", + "text": "Hey Joanna! That's cool! I won my first video game tournament last week - so exciting!" + }, + { + "speaker": "Joanna", + "dia_id": "D1:4", + "text": "Wow Nate! Congrats on winning! Tell me more - what game was it?" + }, + { + "speaker": "Nate", + "dia_id": "D1:5", + "text": "Thanks! it's a team shooter game." + }, + { + "speaker": "Joanna", + "dia_id": "D1:6", + "text": "Wow, great job! What was is called?" + }, + { + "speaker": "Nate", + "dia_id": "D1:7", + "text": "The game was called Counter-Strike: Global Offensive, and me and my team had a blast to the very end!" + }, + { + "speaker": "Joanna", + "dia_id": "D1:8", + "text": "Cool, Nate! Sounds like a fun experience, even if I'm not into games." + }, + { + "speaker": "Nate", + "dia_id": "D1:9", + "text": "It was! How about you? Do you have any hobbies you love?" + }, + { + "speaker": "Joanna", + "dia_id": "D1:10", + "text": "Yeah! Besides writing, I also enjoy reading, watching movies, and exploring nature. Anything else you enjoy doing, Nate?" + }, + { + "speaker": "Nate", + "dia_id": "D1:11", + "text": "Playing video games and watching movies are my main hobbies." + }, + { + "speaker": "Joanna", + "dia_id": "D1:12", + "text": "Cool, Nate! So we both have similar interests. What type of movies do you like best?" + }, + { + "speaker": "Nate", + "dia_id": "D1:13", + "text": "I love action and sci-fi movies, the effects are so cool! What about you, what's your favorite genre?" + }, + { + "speaker": "Joanna", + "dia_id": "D1:14", + "text": "I'm all about dramas and romcoms. I love getting immersed in the feelings and plots." + }, + { + "speaker": "Nate", + "dia_id": "D1:15", + "text": "Wow, movies can be so powerful! Do you have any recommendations for me?" + }, + { + "speaker": "Joanna", + "img_url": [ + "https://i.pinimg.com/originals/80/ce/f7/80cef746e94a1720df6d3ffec883087e.jpg" + ], + "blip_caption": "a photo of a poster of a man and a woman sitting on a bench", + "query": "eternal sunshine of the spotless mind movie poster", + "dia_id": "D1:16", + "text": "Yeah, totally! Have you seen this romantic drama that's all about memory and relationships? It's such a good one." + }, + { + "speaker": "Nate", + "dia_id": "D1:17", + "text": "Oh cool! I might check that one out some time soon! I do love watching classics." + }, + { + "speaker": "Joanna", + "img_url": [ + "https://i.redd.it/3i39xdbz3td91.jpg" + ], + "blip_caption": "a photo of a dvd on a table with a blurry background", + "query": "eternal sunshine of spotless mind dvd cover", + "dia_id": "D1:18", + "text": "Yep, that movie is awesome. I first watched it around 3 years ago. I even went out and got a physical copy!" + }, + { + "speaker": "Nate", + "dia_id": "D1:19", + "text": "Sounds cool! Have you seen it a lot? sounds like you know the movie well!" + }, + { + "speaker": "Joanna", + "dia_id": "D1:20", + "text": "A few times. It's one of my favorites! I really like the idea and the acting." + }, + { + "speaker": "Nate", + "dia_id": "D1:21", + "text": "Cool! I'll definitely check it out. Thanks for the recommendation!" + }, + { + "speaker": "Joanna", + "dia_id": "D1:22", + "text": "No problem, Nate! Let me know if you like it!" + } + ], + "session_2_date_time": "2:01 pm on 23 January, 2022", + "session_2": [ + { + "speaker": "Joanna", + "dia_id": "D2:1", + "text": "Hey Nate! Haven't talked in a few days. Crazy things happened to me!" + }, + { + "speaker": "Nate", + "dia_id": "D2:2", + "text": "Hi Joanna! Long time no see! What's been going on? You sound excited!" + }, + { + "speaker": "Joanna", + "img_url": [ + "https://live.staticflickr.com/3159/2406045813_cab5f8211d_b.jpg" + ], + "blip_caption": "a photography of a book with a page of text on it", + "query": "screenplay laptop", + "dia_id": "D2:3", + "re-download": true, + "text": "Woo! I finally finished my first full screenplay and printed it last Friday. I've been working on for a while, such a relief to have it all done!" + }, + { + "speaker": "Nate", + "dia_id": "D2:4", + "text": "Wow, that sounds awesome! What's it about? Glad it's all down!" + }, + { + "speaker": "Joanna", + "dia_id": "D2:5", + "text": "Thanks, Nate! It's a mix of drama and romance!" + }, + { + "speaker": "Nate", + "dia_id": "D2:6", + "text": "Wow, that's amazing! How do you feel now that it's finished? Do you have any new plans for it?" + }, + { + "speaker": "Joanna", + "dia_id": "D2:7", + "text": "Woohoo, Nate! I'm feeling a rollercoaster of emotions - relief, excitement, some anxiety - over finishing this project. Now I'm gonna submit it to some film festivals and (hopefully) get producers and directors to check it out. Here's hoping!" + }, + { + "speaker": "Nate", + "dia_id": "D2:8", + "text": "Congrats, Joanna! That sounds like a wild experience. Rock on and I hope they love it!" + }, + { + "speaker": "Joanna", + "dia_id": "D2:9", + "text": "Thanks Nate! A mix of emotions for sure. Hopefully, it leads to positive feedback and new opportunities." + }, + { + "speaker": "Nate", + "img_url": [ + "https://live.staticflickr.com/5126/5279947833_4ae1f16e96_b.jpg" + ], + "blip_caption": "a photography of a turtle and a turtleling sitting on a rock", + "query": "pet turtles", + "dia_id": "D2:10", + "re-download": true, + "text": "Yeah, for sure. Hoping for the best! I like having some of these little ones around to keep me calm when things are super important and I'm nervous." + }, + { + "speaker": "Joanna", + "dia_id": "D2:11", + "text": "Awww! How long have you had them?" + }, + { + "speaker": "Nate", + "dia_id": "D2:12", + "text": "I've had them for 3 years now and they bring me tons of joy!" + }, + { + "speaker": "Joanna", + "dia_id": "D2:13", + "text": "They sure lookl like they do! Adorable!" + }, + { + "speaker": "Nate", + "dia_id": "D2:14", + "text": "Thanks! The turtles might be small, but both sure have big personalities. I really reccomend having something like these little guys for times of stress." + }, + { + "speaker": "Joanna", + "dia_id": "D2:15", + "text": "Good idea, Nate! I'll think about it and maybe get pets of my own soon if I can find any I'm not allergic to. Have you been up to anything recently?" + }, + { + "speaker": "Nate", + "dia_id": "D2:16", + "text": "Yeah actually! I start to hang out with some people outside of my circle at the tournament. They're pretty cool!" + }, + { + "speaker": "Joanna", + "dia_id": "D2:17", + "text": "Oh? That sounds sweet! Is it a weird relationship with them being competitors and all?" + }, + { + "speaker": "Nate", + "dia_id": "D2:18", + "text": "Oh, kind of. Some people are more competitive then others, so I tend to just stick around the more chill people here." + }, + { + "speaker": "Joanna", + "dia_id": "D2:19", + "text": "That makes sense! Are you gonna cheer them on even if you lose?" + }, + { + "speaker": "Nate", + "dia_id": "D2:20", + "text": "Absolutely! I don't expect to win big here, I just like playing for fun! You mentioned you were allergic to pets earlier, how bad is it?" + }, + { + "speaker": "Joanna", + "dia_id": "D2:21", + "text": "Oh, its really bad. My face gets all puffy and itchy when I'm around certain animals, so I've always just stayed away." + }, + { + "speaker": "Nate", + "dia_id": "D2:22", + "text": "Sorry to hear that. Allergies can be tough. What specifically are you allergic to?" + }, + { + "speaker": "Joanna", + "dia_id": "D2:23", + "text": "I'm allergic to most reptiles and animals with fur. It can be a bit of a drag, but I find other ways to be happy." + }, + { + "speaker": "Nate", + "dia_id": "D2:24", + "text": "Awesome! There are lots of things that can bring you joy without pets. What else brings you joy?" + }, + { + "speaker": "Joanna", + "dia_id": "D2:25", + "text": "Writing and hanging with friends! That way I can express myself through stories, or just have a good time with people." + }, + { + "speaker": "Nate", + "dia_id": "D2:26", + "text": "That's great to hear! Those are both great things. I'm glad to hear you've got other things to help you get through times of axiousness despite not being able to have animals!" + }, + { + "speaker": "Joanna", + "dia_id": "D2:27", + "text": "Thanks, Nate! Writing helps me create wild worlds with awesome characters. Plus, it's a great way to express my feelings. I can't imagine life without it." + }, + { + "speaker": "Nate", + "dia_id": "D2:28", + "text": "Wow, Joanna, that sounds amazing! Keep doing what you love!" + }, + { + "speaker": "Joanna", + "dia_id": "D2:29", + "text": "Thanks, Nate! I'll definitely keep pursuing my passion for writing. It means a lot." + } + ], + "session_3_date_time": "9:27 am on 7 February, 2022", + "session_3": [ + { + "speaker": "Joanna", + "dia_id": "D3:1", + "text": "Hey Nate, long time no see! The screenplay I sent in to the film festival has been on my mind all day everyday. I keep bouncing between crazy emotions like relief, excitement and worry! Fingers crossed a producer or director falls in love with it and it ends up on the big screen - that would be awesome!" + }, + { + "speaker": "Nate", + "dia_id": "D3:2", + "text": "Hey Joanna! It is a big deal! I'm sure its been a wild ride. Sending some positive vibes and hoping someone likes it enough to get it on the big screen - that would be awesome!" + }, + { + "speaker": "Joanna", + "dia_id": "D3:3", + "text": "Thanks Nate, your support really means a lot. I put a lot of effort into it and I'm crossing my fingers. What about you? Anything new and exciting happening in your life?" + }, + { + "speaker": "Nate", + "img_url": [ + "https://i.redd.it/4q9s9o2607ib1.jpg" + ], + "blip_caption": "a photo of a bowl of ice cream with a spoon in it", + "query": "homemade coconut milk ice cream", + "dia_id": "D3:4", + "text": "Thanks, Joanna. Not much has changed for me, but I just discovered that I can make coconut milk icecream and gave it a try. It was actually pretty good, so I'm proud of myself." + }, + { + "speaker": "Joanna", + "dia_id": "D3:5", + "text": "Looks delish! Glad you tried something new and it went well. What did you think of it?" + }, + { + "speaker": "Nate", + "dia_id": "D3:6", + "text": "Super good! It was rich and creamy - might be my new favorite snack!" + }, + { + "speaker": "Joanna", + "dia_id": "D3:7", + "text": "Great! I love when you try something new and it actually works out. Will you give it another go?" + }, + { + "speaker": "Nate", + "dia_id": "D3:8", + "text": "Yep, it could be fun! I'm looking forward to trying out different flavors and toppings." + }, + { + "speaker": "Joanna", + "dia_id": "D3:9", + "text": "Yum! Sounds great. Got any favorite flavors for dairy-free desserts?" + }, + { + "speaker": "Nate", + "dia_id": "D3:10", + "text": "I love coconut milk, but I also enjoy chocolate and mixed berry flavors." + }, + { + "speaker": "Joanna", + "blip_caption": "a photo of a piece of cake with berries on a plate", + "dia_id": "D3:11", + "text": "Wow, those sound great! Who can say no to chocolate and berries? I'm tempted to try dairy-free flavors now!" + }, + { + "speaker": "Nate", + "dia_id": "D3:12", + "text": "Well I also made a dairy-free chocolate cake with berries on it the other day, maybe you would like that!" + }, + { + "speaker": "Joanna", + "dia_id": "D3:13", + "text": "Wow! That sounds yummy! You're so talented. Thanks for sharing your amazing creations! I should really try making one or just pay you a visit and try one for myself!" + }, + { + "speaker": "Nate", + "dia_id": "D3:14", + "text": "I couldn't agree more! It's always fun experimenting in the kitchen." + }, + { + "speaker": "Joanna", + "dia_id": "D3:15", + "text": "I can tell! Your cooking skills are awesome. Seen any good movies lately?" + }, + { + "speaker": "Nate", + "dia_id": "D3:16", + "text": "Not recently. Any good ones you'd recommend?" + }, + { + "speaker": "Joanna", + "dia_id": "D3:17", + "text": "I just watched \"Little Women\" and it was amazing! It's a great story about sisterhood, love, and reaching for your dreams. Definitely a must-see!" + }, + { + "speaker": "Nate", + "dia_id": "D3:18", + "text": "Oh, that sounds like a great one! I'll definitely add it to my list. Thanks for the recommendation!" + }, + { + "speaker": "Joanna", + "dia_id": "D3:19", + "text": "Anytime! I'm always down to give movie reccomendations." + }, + { + "speaker": "Nate", + "dia_id": "D3:20", + "text": "Good to know! I'll be sure to give you a shout whenever I run out of things to watch!" + }, + { + "speaker": "Joanna", + "dia_id": "D3:21", + "text": "Sounds great! Let me know what you think of it when your done!" + }, + { + "speaker": "Nate", + "dia_id": "D3:22", + "text": "You bet! You'll be the first to know." + }, + { + "speaker": "Joanna", + "dia_id": "D3:23", + "text": "Awesome! Enjoy yourself!" + }, + { + "speaker": "Nate", + "dia_id": "D3:24", + "text": "You too, take care!" + }, + { + "speaker": "Joanna", + "dia_id": "D3:25", + "text": "Later, take care!" + } + ], + "session_4_date_time": "1:07 pm on 25 February, 2022", + "session_4": [ + { + "speaker": "Nate", + "dia_id": "D4:1", + "text": "Hey Joanna! Sorry I haven't been around. I made my friend some ice cream and they loved it!" + }, + { + "speaker": "Joanna", + "dia_id": "D4:2", + "text": "No worries, Nate! Glad to hear it. What flavor did you make?" + }, + { + "speaker": "Nate", + "img_url": [ + "https://i.redd.it/wj5c699jccx61.jpg" + ], + "blip_caption": "a photo of a person holding a chocolate and vanilla ice cream cone", + "query": "coconut milk ice cream chocolate swirls almond chunks", + "dia_id": "D4:3", + "text": "I whipped up some chocolate and vanilla swirl." + }, + { + "speaker": "Joanna", + "dia_id": "D4:4", + "text": "That looks delicious! Unfortunately, I can't have dairy, so no ice cream for me. Do you happen to have a dairy-free recipe that I could try?" + }, + { + "speaker": "Nate", + "dia_id": "D4:5", + "text": "Sure, I know one recipe using coconut milk. Would you like me to send it to you?" + }, + { + "speaker": "Joanna", + "dia_id": "D4:6", + "text": "Yeah, definitely! I'm keen to try your recipe. Always up for something sweet." + }, + { + "speaker": "Nate", + "dia_id": "D4:7", + "text": "Cool, I'll do that. I'm all about these desserts, let me know what you think!" + }, + { + "speaker": "Joanna", + "dia_id": "D4:8", + "text": "Definitely keeping you posted! Love your creations!" + }, + { + "speaker": "Nate", + "dia_id": "D4:9", + "text": "Thanks, Joanna! It means a lot that you enjoy the desserts I bake." + }, + { + "speaker": "Joanna", + "dia_id": "D4:10", + "text": "Yeah Nate, your cooking is amazing! I can't stop thinking about the screenplay, so I just started writing another one while I wait to hear back about how the first one did." + }, + { + "speaker": "Nate", + "dia_id": "D4:11", + "text": "I hear that, taking your mind of something like that is very challenging. What's the new one about?" + }, + { + "speaker": "Joanna", + "dia_id": "D4:12", + "text": "It's about a thirty year old woman on a journey of self-discovery after a loss. Somewhat similar to the last one, but hey, that's just the kind of thing I'm inspired to write about!" + }, + { + "speaker": "Nate", + "dia_id": "D4:13", + "text": "Interesting! That's a deep topic. Love to hear more about it." + }, + { + "speaker": "Joanna", + "dia_id": "D4:14", + "text": "Thanks, Nate! It's my own story. The main character is dealing with some tough stuff: loss and trying to figure out who they are. They take a road trip to heal and grow." + }, + { + "speaker": "Nate", + "dia_id": "D4:15", + "text": "Wow, Joanna, that sounds awesome. I love stories that tackle important issues. What inspired you to this one?" + }, + { + "speaker": "Joanna", + "dia_id": "D4:16", + "text": "Thanks, Nate! It was inspired by personal experiences and my own journey of self-discovery." + }, + { + "speaker": "Nate", + "dia_id": "D4:17", + "text": "Wow, Joanna, that takes guts! I can't wait to see it all come together. I'm also pumped to see how your first one will do!" + }, + { + "speaker": "Joanna", + "dia_id": "D4:18", + "text": "Thanks, Nate! Appreciate your support. Hoping my screenplay gets noticed and makes it to the screen. Fingers crossed!" + }, + { + "speaker": "Nate", + "dia_id": "D4:19", + "text": "Crossing my fingers for you! Hope your screenplay finds a fan and is given its due. Good luck!" + } + ], + "session_5_date_time": "6:59 pm on 18 March, 2022", + "session_5": [ + { + "speaker": "Joanna", + "dia_id": "D5:1", + "text": "Hey Nate, it's been a minute! I wrapped up my second script, and the feels have been wild. Sometimes I'm so relieved, but other times I just feel anxious about what comes next. It's a mix of excitement and terror, thinking about my work getting noticed and hitting the big screen." + }, + { + "speaker": "Nate", + "dia_id": "D5:2", + "text": "Hey Joanna! Awesome to hear from you! Congrats on wrapping up a second one! All that hard work and dedication will definitely shine through and get noticed, no doubt. Hope you've been able to take some time to relax after everything!" + }, + { + "speaker": "Joanna", + "dia_id": "D5:3", + "text": "Thanks Nate! It's been a wild ride. I've been decompressing, but it's hard to switch off. There's always this tug-of-war of hope and doubt." + }, + { + "speaker": "Nate", + "img_url": [ + "https://cdn12.picryl.com/photo/2016/12/31/turtle-nature-slow-nature-landscapes-9a70ba-1024.jpg" + ], + "blip_caption": "a photography of two tortoises laying on the ground in a jungle", + "query": "adorable photo turtles walk grassy area", + "dia_id": "D5:4", + "re-download": true, + "text": "It's normal to have doubts, but you've worked hard and put tons of passion into it. Believe in yourself and things will work out. Here, look at this cute pic - walking them always reminds me to enjoy the small stuff!" + }, + { + "speaker": "Joanna", + "dia_id": "D5:5", + "text": "That pic's adorable! They always look so relaxed outside. What made you choose them as pets?" + }, + { + "speaker": "Nate", + "img_url": [ + "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wCEAAoHCBQVFBcUFRUXGBcZGh0aGRoaGSAaHRoZHRoZIBkZIB0aICwjGh0pIxoaJDYkKS0yMzMzGSI4PjgyPSwyMy8BCwsLDw4PHhISHjIqIyoyMjIyNDQyMjQ0MjIyMjIyMjIyMjIyMjQyMjIyNDIyMjIyMjIyMjIyMjIyMjIyMjIyMv/AABEIALcBFAMBIgACEQEDEQH/xAAbAAACAgMBAAAAAAAAAAAAAAAEBQMGAAECB//EADsQAAIBAgQEAwYDBwQDAQAAAAECEQADBBIhMQVBUWETInEGMkKBkaFSscEUI2KC0eHwFTOS8QcWckP/xAAZAQADAQEBAAAAAAAAAAAAAAABAgMABAX/xAArEQACAgICAQIFBAMBAAAAAAAAAQIRAyESMUETUQQiMmGhFHGBsUKRwSP/2gAMAwEAAhEDEQA/AKBhsOPFR2EIBnI6suuX6gSeQqK6xuOVQe+5JjTMSSSeyjlVowl2xduhEUHIIReZ5sxJ0+XPfpAvtCFW6LdsDOQEA0EFiJk/QV5sckm+iKdvoV8Gw5/aFVGyvBCTrrGhBFN0xluwrLcJLMxm4gEzz88+Yio34I2ET9oveHciPJJUj/5PM0qt8NuYmWRPDTMSsmF16TzqkoKa+Z6C3evBccPasYi2DK3R1IAYfMc6U8Q4Rh0cOX8MDYFZBI77fWlKYHEYQ+Iuw31GVh0PWrbg8VaxlksF20uWzqV7+lRWNw3GT4/YV3HaehW3Eb6ZGsm08aBEUA/MTrNWZLfEHRHLW1bcrEMojbek3DcBgrBK3swZz5HOgUcgCNjTkPctQbNzxTmESCcw6Ejb1rojyX3XudEXasdcLdnIzZQx0bXpz1rjiFw5woI3Op6ChsLhLlxs99WkHNoYHoI5etdthbbOTDMNhrtXSuhg/A4knRlkUTaszoRpuBSEpcT/AG2nsTEGmgt3ghuNd1CyVA0AFNYKDL2FULoBr8j9aEweEybjTt161BgMW7W/Ey6GfpOhAplhoK5g0nodDRVMDsJsu2pBJ00BJ0pbjsXft5iLRuabzP0mmaOZ6CuGJnQ+o5UWgCDgvDrhbxrudpMqgOif1p8XTkw9CK7w1sqwI069KPxF9VgsAZ6CSay0gvbErWQpzAFpPw1zabfNI19Ka2r6QxEAHnUTXl2MEVgADeGIOaZ70VbxKnTSob+CtkeVNZmoLltFIkAdOVEwd4w20/SsxGXQjn0oC5iraR5hryGprtOJWz8LiO3MUDHbpoSSQBRHBMdlBABy75iP1oG7xUBcvhsc22lMrV9vDHkyrl1mh2YPxWNcr5F5bmlg8TJ+8fXeBtXT4q44UWxy947elR2MHdMlmWT0FGkYj/Zc6S9xhPQ1EjIgW2raGe9NbWHLL4bKI60PjcLbt5SFGh+nehxXsa2I+J2sjhzJUjSNNv1rjLb8oAYsxBJnlXPE8UubLOYQfvSvCcMthszXLvoHIj01pUqCXSyjqoCopMbsYpZaxRAa2xzNJjKNjQmGwqtEXLog6eY7UdhbCYfMRJLRvqZmi270YCCuJBOs1lM7nFLYMNE1lG2A8H4TjGsv4wUmAwnWJI3mpmZHSTmbEO4PaD35cqv6pYuYa9atrah3JtiRqCog/wAOs1TR7K3Ektdsr1Jb+1ccWp76Fbj7jFsXZFsDG3PGcDKtpDPhxpJjQt3pbiuNplFu1h0UKIDP5mP6TTt+HW3toovWPFEDPM7fma6xGAu+GUfE2STzaAfTaqJquiXKJV7+JQqoyPmBk5mlT/LyrjAcRuYe6LlskA+8o2I5qadWPZ8yCbthwNx4kSOkxpU1/wBn8M2viLabtdDr99aNxWh04oLx+Jt4nDs6RtMHdWGsUo4Txy9YgISUHw8qnXg9y3adbZS6rE5riNJyxEFdvnQljjd+zb8NfDI1gsgJHoaGKLin7XoMElpFys+29ow751aNuR+lZY9prNwlVZgznbLz+W1eaeOTqw1O/wA6unsTgRBvHJB8ok6iN6q2kVReEAABIkiJNd4jEzbIiFO/WOlB3MYFXVgOsHfpSe9xAg5V17ml5hLDhsaGIhQoGk0wBtHV5md1qs2bk5WA02Io18SU0j9Zp00KP8sDyPmEbMP1oax4mczlEDlzoWzi3jTQDtUb4ps0mmAPsPfLEhlkR9K5xWDUKrAtM6QdvlS7DY8iAdJplZxExrtWZjWAwhUxMg6mRzplesKATA/tQi49ZmJHWtPxGT7ulboxsYQEiGIWNRUGP4XbbIOX+c6zA45XZiZGWdDpUV/GzJE6f4KPZtokw3Cbe2Vc0yP0ro4UiZdYOkRz9ahw+LPv84qK/jGjYATNYGxigtiB8Q2rWMx4VRpSFsZBzTvQOI4mpGrc9t6No1Fkw2PERFabiRHryqqW+JAMF82vWib2KE7igYsv+qSsDRutKeIXBAZ2IzsANd+1DNi1jddutKOLXS7Wy7JktyQ+aIOXTT5mpynQyQJxjGW7eIyyYMfepBx23mFtdYMEmqSmIL3ma6S66ga8uWtG4K6pceXKBz5xRUgHqHDrqwDNTvczOGJmNpqqft9tAqIXJjUgTQWK4pdIi2p+dHnFeQUy2XrSMxLEST1rVeei5fE6nespeUPcPzHeLxV58lyVSBo2iqT8Xm0Vp7EmucHezk+JctMWJ+JYLHvmINV7E4u5cuEgtLH3cxJk8hOp7CjsNwO8crXStkN7vikh27LbANxj6LTRi0vYS2zni3DnRvLbOXcRqCe3X5Uoe6xkHTsd6ae0OE8BlUM/iCQMsr5Z0Yk7T+ATHMgiKW4biN1Cc48QbFboziOnm1HyNMgtArseVbljz1prat4a7s7WW/C48S38mUZl+YNaxHDHQZvK6/jtsHX5ke784rJpuhbA8JjLls5kYqR05+o51LjMZ4hzZQCR5gNp69pod7sVDbxJHei40wr3JA4PaiExLqIVooS5cEaCDRGDw73GVLalmOwH+aCskvIdlg9m+K3GuZLjAgDTMJ9fnVo4hjrdoI1wqublOw9Kz2f/APGu1zE3AkakA5QPU7/lU/Hhw21Nu3ZS9c2DHUfU6n8vWpvHydIopJLYVhMVbuLmQh17aV1imQLqQOe+1VRcW20R+FFAUfah8S7aiQW5k/DTxwV2xed9Isg46EJCGTHxQAfmTFS2/bC0py3bDAjeI/I1TrdqTmbUDYHYxuT20+e1GvcY+fOQvpvH2qvCKETk3ot6+0eHu+6cumxEfeprF9/hIadvNEV56uMh5cAqQRKqoYdGBjcf2o7CcQu2QHttmtHn8PKQQfdOo/vvSyx+zCpe5frOLtqf3kpI1n+orbcatLbfIGYqDkAEk9garmF9oUubgKTtpK5uh71NhOPLbUs+Q9l3HyqErjplItPoY2OIOtkXb5yawVO46DTeoRxq2yEqHzchGn3pBjeLpfuIEts75hkHfsBufyo18I+V7l18lpMyjLBN27GiKT8I+Jh6CdDQjJ9GlQ54dxUKMtx0GbbXX/NfvSri3tAEVlDhiCYWfpQmJwj2xd8QIvh2w/lB0cr7hLSIQsGYA75BrVOXFW3eXbUnUxt3pFmUrrwFIseH4uWMXHfU6BRoBU+GxTBgttDcEazoAT3pPZxNlfIjNc00IFFHDO3+2XVeesflS+pG7YXFhN3G3RcLuoGUaLM1Ff4s9w6Ie+m9EDh4EH4u5n86Jt2Y/tS/ql0gOHuKVsXrurCI25VH/ogJl3+VWDLyqJ7VTnkk+jJC1OF21GgzVLbt5dlFFXEjapkTQHc1FqUu2UVE9h1jUCakYgbVwidRW7yADSqJUgMEdlmsod312rKnyQTjiGLW3bHh3PBU6ZcKqh20Mq12Fgb6+Y6HfWkljihVYsoLU6F5z3G9bhAI/lC0VjOD31t5RaLNOgkHIBmJbfc5oH83ag3wVxQM1t1/lNepGUX0zm5J+RbxG+yqACY5id9d4696XJezEltSd5orHXM7FRyoIDXUfMVSn2azbpGq1ILrqQVJBHxAwR8xXDdjWG4dhWcfcFhjcQV/922GP4x5H+ZAhv5lJoS9bUGUmO8T9t/tW2Y8wKktYRnMBTrtpvQUUtIyVAjMWI616n7AHDW7LMYFxffJIknlH9Kr/CfYjGOJNoLmAAzlVaJ+FXYEtUq4NUZ7COlt0JDpcm2+YaGZBB5GQSNaVTjfaH4yekhr7R8XxN1vLItzAVdY6SRuT1+lJRhbrAuLZMHXqPl0o61wXEMCFe2AY2uTP079ak/9euEZVa1OhID8+8CZ1o+vij/khv0+V18rAsNmtgl1ObXcGddielTWcG7mSpgAsYG8DU8yT/Sj7HBr6yrX7aT703TMCOo0M1K/B8Rbgm/bKiWlrpWCNVnNuOXzoLPiv6lsZ4cjXT0dPwW3ktXGuhmcAm2muVY8q76EaA/OpeI4VFQLaVhpqTB/QQK3h+EY0Av4lpg06yja9ZYa/MnSuMRYuEhbmLs2RAkWyC5POCI0nlNBZ8a7knQFiydJPZXMXgHGrDKvNj7p06xJPQAGu7LhUe2okvEF1EEqRAjvGhJ30gTU1/G4S28K9zEHmBAmPiZiTqJP1NL8dxtSCtuzbUa6t5z1Pvafat+oUuk3+AvA46k0vyT4a4pJt+HbFyJiCMwG438rc9PXkaFxFgXri3FzKNro7rznkTsfSahwlzFYpw2ZlRRqyqFOgEjOoH1nankOWGYZjGbKSNdB5nP4Y1AO868pLlyW1+RHFRfyv8B2GxVq1lCL+8ueVYHmW18JIPu5zJ65FP4q64TxK3cuTcbNh8PadrYeSGuMQVmPxNsu8D1pPj0W1Fx2LZ9TJys6n3spA8lvlJgtoAI0Fbx3EHvMFBhBsAMoE7wP1Mk8zyqDbknFf7MkuwzjvtBdv5lZyULlyBpncn3mA3jkOXrSa3ZMbUy4ZgVuMbYGo1mmo4QwBnalbUdIdRvYlwFx7ZJin2E4xcyxArDw/wAsmIoXwCo0oRxqbtoE5cVphz8YfoAans8a092TSJiRrANdBiu/MdYoPFD2F5Md/wCu/wANcLxiT7tJVYdPvWwxq0McWuhZSdjo8V/h+9E2MfmjQk0jw1uTJ2pzw7CksWO3ITWligjRnJjSxijGoqPE45RpW3SKV49BB0qfCLRS2R3OKrO1ZSVwJ2rKX0o+wvNjTiq+OLVy0xV3DeQMcpdSM6A8niGA5hjzGqR8U6rGdgx094j1pyPZ66LmSQRGbMje6+vhsQYO6xptmpTxbO7qbyNbdxoWGXzAwyMDyMSDyzAaja8ZRultE7XgCs8RYmLkXF/jEn5OIYfWmC4W3cMWic25tN738hGlwdtD2NIXtlT6f5FEpYcsGHY6cjV4uugSSJmsoCTrB5DrXfDLP7yGEId53jt3NGtZF/QsFv8AIzAu9FbkLnRufPXWubVq8kAq0zBkajtR5WKmPFXC2yjB2JZdlKgjcSx1K67/ABdhRKXMNaRmW+xuHnbRi3/JwsD+lL8NwRmIzqys2wjU0/wvsg8+/l/+oP2qMpOWot350UWtsj4RwNLgF3xLhd9fPHkBOu8+bvuOtGce4J5Ree8btxBlUOJa5mby2806kEwCdhT3B8JNuM90mRuFC/TrVb9rcCbt1bdu7ktI3muayXHQDYrMAcye1cs8WRSu9F4TiV4YbFAlhZuKOw5fKpsNgXuee2chB8wbRlbTXTUimvCsLxBbgRXzW+bOc4UD7zp1+1OOJ4J8QuVoMc00nXmOYqP/AKxltKn5W/wdjyYnGr2vD/4LMfwB7s3EbUxOaYJ01kflXWK4IfDth31QMCXICsD66aaV1hi9nyMWAHJgR9P6VrFY6w0Jca3rqA5APrrBpOclJJbS/wBnRH0pxbem1/AJw7hz2rNzO7KGBAGYGAJ8wykgb9eVDYT2cF0F7rkAKji2oAY22ZVBLHVdGUxv5xsZqw8Ku4FcwJUholQ0gwInWZ9Nus1LxC/h/DuZGbMUyhxMkKFygjb4E+laGVrI3T2/Yllx8saiq0nuyjcE4Jbv4hlFxbVnMdWPLUqgLEZidBvOs0QeC2jdUO7XAYnKqkR0EISV7gmpsO4tvaG6jxDcX4WH7saj159qsqcctqrBLaDMI90evrXRmzShKoqzmw4FkjbkkR4bB2FQ5WUgDMqmAFG0pbOtxpnVtBp5W5heKbzm1Y8NcoDMG87TP+44bW428BhAzHQc5E40VM+GnpH5xSbivG489vKtxDIKABgRuJ6dR3qXrZZyqqv79FZfDYoRb5X/AB2G+0XstNk3FLlwSxdySXMc52Gmgrz9rNwbqRV7v+0D37aEsfMNR3GhoN8Ky2/FKMUmD0HStizz2mujnnjiumIOAO6XlIB102q73VcDzAZepMDsJPPtvSmxeCkOqgt8IjcnQD1p/hcAyP8AtGJcMtlfEIYyrXTogVPwqzKJjWCdorshc1yaojKVaQA9u4ynyFQSAAdzOgIBjQk79FPUCu+Iey2ItrnIVkIBkGdDsfSmGGVXXCuXaLouGT0W6wAgCfKFzAnfNGmlWm5jh4QssvkVcoaZhQNB36VOeZwdBjDl2UvC8FsIoZ/MY1XpSDH4QFz4akLOgp0+LhiCjRO4qZMOm4HfWk9Z3tAcVQgHCnI5Cul4M/UVYS68yflXDMDyNUWZi8BXY4URuabYbCMOgrLJnlRK2m31FZ5HIZRox8KYpVjcK2sRTm6tw7QO1COhnUil5tBaKtcwLTsKyrKUXpWUPUYOIudbt1mbDlWWSVs3QqOV1BtjUFyCDBBDQNSZml44nmR1bxLDIyhkug3UOcNAK3PMo8vU7jrTFku57lv9lmyGZ1GfRpYEA5yYBGkQQCAQNIqTiOGW4jJduMbTBCHYKbltpUEmGEgMxzIDAiQDmE0Si9iyURHieG27g8VrZybG9Y8ybfHbbW39qy1wTy5rbeKo/ATI9UOo+9R4ngmIwrK4cG20eHfttFu6NSACwGbbVT33oi7gbi3XNtwoDsV/eIkCTGhIIotPqxJ14ObFu3mOYAkdR+dNuHYwXHyvBuA+QzGcDZT/ABDkee3SgDduT+98K4D8WdM//JTJ/mBqw+xfB7V17lz8AhQ3JjzkHWB6UjxyldsWIzw2OCXJYSBvOp9B0ou5xdrmlu35p0J5etbxfC2Viq6k6yddP60g42t21bZrYm5AA5nU6kLzNdeP5Y0hmr7HX7bbsy16+mccs0wegApHw7GYds3j3xbbMSADDecgm5LAgEiBpy9aoDYa63/5uW3PlYmpMT4zuC9t8wCiMp2VQoMegppKMqsCtdHqtkiFVCotLtkIgjuQdSedHjiaFhbtqbnIZRr9P1Nec+yXCMZcueQtatAy5cHKewU7tHOvSrmJtYRAojO2gWYZidix5Cf+qjiwrG3Tbv3/AKDV9jJMIGBNxRl/Bofqf0H1pHxS1w0Bs9mySN8qAkT1Ye6fvVd4l7T3LsrJy9AciAd4ln+ZHoaSXcbcA/3GHTKco+QWB9qt6V7YeVdG8Zw7BM8paKqdAM5gnr+KPptUCWbFth4dtCd4hm/MmflReFIcN4iu7R5Sphj/AAkkGR9xr6Viuq+ZbC6aeY+KT1kHQD5VRKK8G+Z+SW1ct3GmMpEy4BZFnWGmcup3k+lY6c5t+s5R+VcWL63UyQLToSwRJCsOZgn3xv6emoruttgpAZW36DbpufSg8cH9SQIykumxlasIVkqVmfMrqRI6SSY1G078qrz8Lug3HIFwFLnuanM0D3fe2jlyNFWX8ItbefCYyIEweTgaehHMfI1ILr2iCGkbgjUMOo7afLWpfp8abcVVjPJP6X4JuAvhrFtfEttduEZsoB8pn3TPOt4m/exLOvmtWYnw95jX/BRdviqt5tGnfTXv6etT+NbuBzbY6KQVYQRPWdCs8/qBIrn/AE8lO/DKufy0D8Ly2bLXiPOPJbUj3XcEZ+5CoQOk1DcFxgmGVyRdNs3V3dmksAGPYgx6TtU2IZFtHMPNJyLsGYWrgTUaxmcHToam4ThMhNxjLuoALCSqZAojkpZQCTE7Dmaec+Nqv2JpWMeNW7Xju1mcihbafhCoAIQD4ZG/MiaBN1joWMURfDf9UI9htwZ+dczjGquw27NkxzNbVl9fWhXUg6k/eu0foPt/WotIcKYDrW7Z7UIZ3gH/ADtW7P8A9AVTHAVsY25FFWQf+qDw7zR6pH9KLVMZGXbpjb70FM+tFXEP4VoW4vb9KRsYEuIZ3NZUbuZ2P1rKS2agfjClWFxouFUIJHxWwAS6hSNBmUsgOnvKdCK1wlMMbb2rhZbdy34jyQberCLisEDhwWXQiNBMRRuL4UP2RDauNbazclSdcs5ssnmnmKEcxBO5FKuGWUW6p8PKlwZGtn3Uu+JayrzBWXDL/C8V1KVoSS9jtrN+0r4cB7uCfK4ygOFzKGW6MoOVlnXQBoOmuiziSA3HVmBGbcDfb6124tooUO7ZZyuDBy5mjynbkfe500xOKu3LgVLisfhS4oYGFVhGYEZjJH8sTrVE97JNaK+tg/0M1afZPFNbLZgQrjfow5/f7CuhwxLkXAyq8S1pdQSNSU1kd05ctNBqy6H8Q6j9Yp3KLQsYyTLPhuJPDSASG3ncQAKd4fwXyuurRrPKeVUWzjktObbHRgWBO3cTy/tTq3dU6KBG8g8u0a/91H1JJ/YtRaP9OtnMTMsQeX5igsdbthoZQCFneIE6k0kv+0hsLq88gszsPSeY5fSlGJ4k99bniCAVzqs+Zogy2+UHkN4q2OUpba0LJJdBfEfacAFMPAUaG6dp6IPiPf8ALeq2cWS/iEF2kEM5Jk9co/MkxQ1tGe4C0HkBsFHQDkKZ2+DO7AKJ5nlp+g2PzrpjRNr3BcPattJYukR5QM4JjlqNNNfzM12LNrMJS42vO4on5eH8t6JsYVF1HmnQRoDA1jXXc024U+HOZL1vLlHlZM2pj4gSST9N6LbMqCsHw3CNb/eXArxlVSdQRMgqNJ830IpHi7AS4wtzlnRiYDA7EyNBrsTWYy4Ga6VkjMra7kaqfmcy/QVNhrjNbyq7m2NhMheqwTod9O9LGLXbGb9hFjUKXMyjKdGUg7Hcehoq/at3FW7r5/fjSGEZgAfUN0gxyqTidgl82UwVG+m1D4VSy3bWkr+8XoYHmXWN11/kpl0aSqTOcM4hrZEsFaAYIJKkRJ25EfpQeDvaeHc90/EdfDfYE9tACOw6CjLCqAzMYdB5PMASZ2IAk86mxnCc1s37eUyJZdo668v71Nz4yplOHKF+V/QquK9tzmGo3j9I09OVELjChFxN50kT6g9jNTYXCXLtoxMpEGYzWzII35Hb1Pajbns+LWHF+6YMhskgT+HXuNef2oTkk0CN00QYsAXMqt5SPdPw513UncgNE6ER86WcI45iGvlHKZSzaFdE1MAFY8o21ozHPacG4FKsCWeXzQmwWAIB2Agz25kL2RdFe7dYAt+EOyPBaTEFSdcskNIHwmpZJRinLyaKb0XRHEakH8qGxQiWU7bjtXaXVFtfdyHnOnZZGx7ETUdxGXXKcraa+ledmnK7LQimqMRQ4zA6HlQ9zCgGdBUVhmtvkOinUH8xRd5M2xqanTNxIbaDmKl8EdBQV1GFc/tLDnXQsqXgRwHeGtWyhZmC9NJ+9buXrawfEmempFIDeO0mtpNL6jYdJDe5ipHUdxXCYgHn9opNezKQYJ7TXT4x2PktZfSSadU0LsYXHE7GspacTc/A3/GspKGseex1xmssjn3Q1u58UERkcg+8uqT3AnegbFu5muWmU5swYKZhLiOcmUjXKrSCTPkYNrBgnA2hbxJttcYFyG0SMzNm8S0SrEFGDGOY25Cm2IFnKpLMGcErmBYuo0K5gASNNVgk6E10NLwZPRUHwDpdBKBkKy4BBy5gc9skEgPuyttOgJIobGhhcJW4FhkjTXRFOcRsNiPWnvEeHOIuW7toeJcBbOz292zLlJUZXZRqOZLn4qj4tgobxGQM4QQVeARoBcGhkKTDCBpBGmgb1OKtiOF6RzZ4iLeVsvm3JQ5RPWCpA118oFTvxG3cbMVAJ5LpJ9IpthPZS1cth3xJkgaBdM3Ma6nn0qfC+x9tfNLzvyEf8tY+VGLU0nFP9wNNabKtfwy4hiP9sLoW3zGRlXLud/TShEwWJsBil0oQQChEETOWATqDHwzvXoNr2UsgEA3IYySHn75RFbseydi0rBVDPydxJB7RGgqM4Zttq34odOOkit8V4fcfDWbwJe8P3bGNBJOZh3BAUGlOFcqRbtxcuvo7HVT/AAieXVjpp86uNg38O+R7Yez5vMnnILGTmX3o32nltSLiWFtWmf8AZnztcEGVjIrSSoJ2YxHYE9av8Pka+WS2JNXtA9qzZ8TyK8nmpEE8yAwnLI5mmmIe2oyM7nNEhBCtA6ZtY9KUYa+lsibltGYhQPEBbMdI0JiTOpjauMT4dtpuYhFbWfPnb6JmNdSnH3F4y9h86YS4ADKwOiwDH8TCW25zWruHwxU/vANNmB27hWIInr1qvX7aFlYX7RVoKr4qhtdpRiCDGsEVy9zMGU3rWYwFBuZTAMnzNAJ25mtzj7g4S9iw2MNaTz/vMpEBwqxMg6AnXb7UMTYt3ZF+AwEyhOhE66Qd50pIcTcRSvjW1BjRr9vWNRpm9K1irNy4qXEi55QpNphcykTvkJjSKPOL8oyjJeGPilu6covW823MA9IBgGl9vCWLNxs9zxGGZWVdBqCpEnQbnUE0ttWrgKk2rkA6eRhz323qXH4S41x8ttmDtnkKx0bUCBtExStquxtt20S3Fwrq7nxFNuJ1DFgTEgZuvOuf9Yw/urbuEGJBbQiRmYqCQTXeH4Pc1VkCo6lZdlGU7qTrMSF2FBrw63bJFy/aBn4Cbh+SgafOoyzY+7uvbZSMZrSVWa4pjLjXHt6KikhVUaKBs0cyQZnfXTSKmZG8AHEFgEabcmWbUDKEO46HTc8qHxHFQrDwred1GXxbok6aCE20HM60Fjzcck3Lha53OwO+3u79KlLLKTXFUvv3/A0Yd3v+hzgME2OuLKLaw6k58pVS7QNNAJOo1irRjfZm3ctBclu3iFXysihM5HwkgSAYPWOUxVO9neOfspHlz2zoQdDM+8O5/pXpGB41bur4iERpJ5jqCNwao1HHGmJ80paPL8HjzauMhzhZI1EMOoYDQ9KepjLYUEupG4M6/emXtBwdL103FhM0ZtBqwA126Uq/9fRd2k8tR/SuOUYS2mOuUdEGL4xYMrJYHoNfrUS8RKe8GCn3TEyKMTgtsnzCR6j+lN7mGDJ4Z1WIHOOnpSNQivI3zSK6OL2yYM1HcuKwJSaYHgyJvbB6Hr61w6FWgrCbAEc45dqElHwLvyLLDmdaPtelDMhUzH2ipbLE6azWpPoHRYMBgrBAuXLgAB1QTJHSSIptiOP2LalbFvJ1YKsntqfvSLDYckVI2HGxqsbS0Czf+sWTq1lyeZzf2rKFOE6AxWUKNbOrPD1W+4t3nloaASxUvkklcwMZiNehNOr6W2tW7dsOHQ5g7KCVaDmZRzBgc4M67EUNi1zXFFi8BbVgzA5TnCwYUg6DSNRQFrhtxJ8O8CbS8zEFmc3GKzDHzjynse1WSd9mtUS+PcusLYtnYC9oQBIDKy5zqoM+WOfOILYYBLhAGYG2cuaSBA6KDBkQNgNBvqKVcMx1y5Fq5hlTTV1uOWBCltVO7SAMvUn594LE3bZ8O4jKdwxkSCd/00oxae9UjNlwwtlFAhdQIzHeOnp22rdy4C0KSfyGn+fekqcTQQpM8/l86K/1S1+KI117jTSumLRJpjW0hG8/996lVOZn0/Wg7GJLCQTB2kdjGhrnEYoqAdJ2A/oeXKi6MrNcZuLZttc0n3VBO7n3Rp3+0153ccCXc5mZie7tuf5edMOP8UNxybhItWREc2uHcesQO3mqtY/F3LdzxGIW4VGRYBNsa5dDopjrtm2nYNpfuarC8ZFx1tuoZ7kkqQC2bUgFd1HODGmu1R4v2RukAo1sDlb1yoJ2DfQkkCah4Z7KYjEL4hYIrHMC5Msd80DU9ZNHBcTg7n7x1cRCOScpMz5pHy667mIrlzLLpwrR0Y5Rj9Sv+QO/7HXlUzbZwYOa2VyiN9Dvy5ihLns5ejM9u/4UaEqzw2m5ClQu+xMUcntPi7V1nW4nPKMobQiCJgGK2/t9dmWthj2uMo+msUq9dJSaTftZVLDJ1bS+4sw3ALRUO105CfgjQd5WQZ0260LdwK2jnRiTOgJyEiDpKtmnbYUwve1KOSThEDfiFwhvqqg0Bc4nbb3cOoPVnd/zIqkcmRr5o/lAljwp6n+GH2b99FDeK8k6obrNC+mbbpJnSmeFvF5JZ55gu0H01qrHGXJ8rBOyhZ+pk/etMbje9cuf8o/WlWJu7rZvVjF6ba+5b8QVIg3IA5f3NAXeIWLYgFWPbU/baq1kWY8x7nWphkUagTy036VSOF9NizzJu0thb8TU+6kD6fczQ+Idn12gyFmPXuTU2CsNcOW3bLkmIHL1PIU2u+xuJWFIRZgxmJjNtMLp9asscY78kJZJSVeBbbtg24mOY9alwOLewc6vlOxkSDpsRzBoLimCu4ZxbuqVJ1HMEAxoR+VQtis+jGOn9aSUlLTQIx47TLNhvau5fuW7TBcpIAgEQeR3q1W00gx2MGvOOFYYm4jpM8vU7R0NeiYNHVBmLk8wxn9fypfSjxqgubbClws6x9KjdI1FS2sW3ISNu47610+IAOoPrH61JYm9Dc0cW3keYGgeIYGVDrrB0FMhf0zbL9ftQ+Muh4CTqfSg/h6XYeYhuoM2rAxy3NTmym4EHnXeNwFxBnB03iZNDWGuttlI5EEVKknpm2NMKfpUjrBkbVBbwjKAXgnpmP8A1RFhyonKIPMbfnVFaF0DizdEhU0nSsosYxuREfOt0eBrJMZw23kS5bdx4jSxKZmzNEtA2HKIjn2qw8Ut2GtqoyEAQGQgNtl0I17d6AsWbbXDaa5kuxmVgd0HMBtCo6enrUuGwV65nzZQusneV5OhIlDzg5hPOpKE3HT2G0LeC4EeJcBvE3JkkASVPuyXBn68qN4qqC2VZvEYCRoMw0jcetSYHgWFt3C03HcwGzMY02EaAbnYVYrmDVkyIqgdIG/9avjwyX1MDlqkecrw66VELJbWDAI02ipE4BiCvuCdTvr2mrXh0y3GtuuUgZlnY6akd6ju4pVcBB+75tsZ3I7118K6JKQgQXbQOZinr/XlvXNrH65C+cR019O361bXQXBDDQ7GJqvY/gEAi06o5MqSuYfQmoSjLwPZQeN3l8MEsBDAgDd7ja3HPZZyjuWpXwpVuXUVhpOZiRPlXUz67fOrvc9hA7hrl8ZAsEhIYkzIAEAD760ansnh8IpuW7hKtAIuEA8zoQNu1C2v3MF4PjwZotoI0UHYE9umlCcaz3bbItstmmQvmgdQRtrtPSjLXD0cALaVAdTrqVHMg/maPu8XFki2EzADqAJ+kzpTqLNZ5NhvZ/F3GYCxc8u8jLvtE7/KpcR7GY23lz2GGb3fMv097Q+tel4nHW3GYNDbAAmfSP70G/Fr2RrYl0OuUrJXuvMf3p2mazzZPZ2+THg3J00K5fz5UcnsXiX+BU9WBP0QHv8ASvQkxCOfPmEe9PX5wPvTDhmE8y7eHOYAEEkyOk6bVGcpRfQy2U7B/wDjkJb8S/dO4UKnlmdvM0/lUr+xFhGzO90oYKwVUgadQZO/TcU14pxD9rxglh4VhyttdPPd2e5psF1UdwTTWy7XXa2rEfDEfUmD+VCOVOVWbiVxPYvBtOXxdAI8w1YkjUhdhpt3pi/s7hbfuW0NyNSwz5SBrqdBMdOdWTh+FcLAVW6dRB1knWee1GJZEhYXMdQNiRvtG9VtC1RW8EioGXwi20HLpPUfOaIIe4zBylu2qwSIBzTop5Ef1qw4jDIBJKjry+X9t6RLxK2M5UMSWEeXmZETzO29JPLGOmxowcuiv+1XDbOIw7BSGIBa2wOzADbmZ2Pqa8jsPlMZSSPxbA+nOvXuKY4C25yhVQ5mgD4CTl78q8t4fgmvXGOwJLExoBM0mPLz2/A04cdEnBFe5iLaySM0n0Gp/wA716Xw1iTkJWB1FUvAWxb0S2W/ijU/IqRHanOGS43vIFB+IkzHpr+lX8W2RtXSLV4CjQwJ/h39DW0woEyTPXLGnrzqv4a34WpdiebE/ajkvA6EA+pP6VCeTdIpFEj4xQMsnft+laDq0NO3+c6Fu2mM7AdAI/LeuEOXQxUpSGQYuIJEN8q5dNBlA9B+frQviHkQPlXVm8VM6tTxwP6kB5F0FeKcsHnW0y5Y1/L/ALoHiFxx5x7p97QfWord2dZYn10p5KlYi7oYhD+M1qhP2hu31NZU/UQ3EsHCeFB7q3SXLqTDFzJB3Qg6FflyFPuMNkQMDDKwgDQkcw38McqT4DEwQTvr9Ypxh3DoVfWfLPMAj/quqEFFIm3ysCTEq2tsKOZkbEzv1ozDY3KCLjCSd50ihcLggqlG3kmT25UpdXEjuf7VSULYFOlRbL9sXsnYyCP0pDxNfDuQJzLGhG87x2IrvB4koA2hOm5nbpRnFrudFcLDg6HlHT571NSa0O4rshw/ERopI6x1mmF3DJctjXvPMGq5iXDOjKoUNrHrvTXBG54YVGEjedeW1Ukq2hIuwJlY5rZ85WIadCOXrUrWbRuLcdGcg5kDGVRtpC7Tvry5VmLDJctsyhSfIY212M1IuJFtgH91vpm69qWk90Z6OmtKdiQ56HWOR6Gl+I4QTqN/incz3qHiuKyXDl93l121H60w4bxFlKI5kNoCTz5Cs0FCW5wp4JIIAnaNv0qOwG82VWzDTPmg/L6Vcb2HzZoYhSNY/XmaExHDkcABlCzqRzjWsYrWdxmX3iQNCQ0gxzNSIWTQKVOh0P17fpVmw+Ctocw103InQ6HflQ9zD20EqoPziD17VjCrE8Jt3BnZCrGDMZQekwdPXShrFq5aU3FctcDyApYtGnvAiCIG/brT5MVKjlzI6f2qPG2Sg8RfeAOi9Nf8jvUcmKMl/wBGhJpi+97U22cMzMhKjOADyOmUjXXXfvQae04Fz42tyIYMcxIMj3uQPIb/ADqnK5uNcfOBbB8sqNZmTA6QftUtu2tsS7DN7wzIOY05+UffWvHy8oyab3Z3R4taLhiPbM3UYLbbKD77QqwNDl3YnQxvzpFe9qhbnIkb7sPeJOoAHSJ9aVtf8q+5JG7QAJnqNB9aS4t2k21JciCfDIYdvN0G8DmaePLJLYOSgtBOJ4revq1liFUsWaB5ieQJHLXbt2rpLZW2ttBMxmOpzN26CpeC8Ue35Fw6ssyc0EzABYEiRoB9Kd/s11nzZLSjlEzHTSvQxOCi1X5ObJyl5OuHo4tqH36DkOQolkC9Y5wZqVEIGsT22ragzqJ+WxqcrvSCkqIVCn0o6zZtEa3Mh/iMAfPKahe0D1Hp1raWj8Qlfz9O9C+PfZqJjYtqCWuCOUHNJ9DGlC5EaYBPfStPhbZ3H10ND3sPbBkaH1P6UHKP8mpkmIJAgfcD9aCGI80MfvXN/EE+XMTXC4Zj005kx9q7vh1Ublo5su3UQ5Lw907GhGU2zlnTlry6VtLuXZxMRoP12oizdkZi1sR8LtqfQDWkypvroaDXnsBdbhMgCPWt0xHFrPNbM90M1lc3H7lRth2nQ6EUYMcbQlvMDr9K3WV2w6El2cYX2hW45WCK6u4tcwAGh11/OsrKsuiL7JbFslZ+UUYlshSxJy6Rr+lZWUskrHi9C1pD9YB02gfrTnCW0VcyiDEkfnWVlGfQIdgd+8LsFhsZ+YqHGI1wgGB19I0NZWUowr4/iAqGf9xcunIzpM8ulKcML16GLRlggcgRqDWVlKuzPweicI4il6ySg1QlGU/iG+vMbbda1+xsoLho0kj4Y6R0rKyt5GIrbZQHIzM0czA16E1rEFWHl95gd9pG461lZWMC/s8QrakztsNe9dcVvrbssoPmYSJHfWTWVlJNviwx7R4mLjo9wo0BX0nX4jl7EaTBFNrivist17pYxA8oGg+XWaysrz/inS5LsvjJLfAlIkksQObH9aJw/CEzQqify+dZWV50s0/ctxVjAcMBMkkPsTvI6RtvzpoqQB1A1+lZWV1fDyeyWTwYdP761E7tO6xz0M1lZXXHbE8EiNInatrfI0k9+Q+1brKfgqFtiq7jXOoy5eUgk/58q4JaPeP0WPnpP3rKytCK5Ak3Rx4zRyP2/vUDgnppyrKyvUhjjRySkzkLB3JHQaVu7lI0msrK5PiEWxi65E86ysrK4yh//9k=" + ], + "blip_caption": "a photography of three turtles sitting on rocks in a pond", + "query": "turtles basking sun", + "dia_id": "D5:6", + "re-download": true, + "text": "I'm drawn to turtles. They're unique and their slow pace is a nice change from the rush of life. They're also low-maintenance and calming. Check out this moment I snapped!" + }, + { + "speaker": "Joanna", + "dia_id": "D5:7", + "text": "They look so peaceful! It's amazing how these creatures bring so much calm and joy. Is taking care of them tough?" + }, + { + "speaker": "Nate", + "dia_id": "D5:8", + "text": "No, not really. Just keep their area clean, feed them properly, and make sure they get enough light. It's actually kind of fun." + }, + { + "speaker": "Joanna", + "dia_id": "D5:9", + "text": "Sounds great! Having pets must be a wonderful experience." + }, + { + "speaker": "Nate", + "dia_id": "D5:10", + "text": "Pets definitely bring tons of joy. They are always there for us and they're so cute! Relaxing with them is a great way to chill." + }, + { + "speaker": "Joanna", + "dia_id": "D5:11", + "text": "I wish I wasn't allergic! I would get two turtles today if I could! I found out recently I'm allergic to cockroaches as well, so who knows if I'll ever get a pet." + }, + { + "speaker": "Nate", + "dia_id": "D5:12", + "text": "Sorry! Maybe there are other animals you could consider! In the meantime though, I'll be sure to send you pics of my turtles so you can still watch them grow without getting too close." + }, + { + "speaker": "Joanna", + "dia_id": "D5:13", + "text": "Great idea! I'm already really invested in those little guys!" + }, + { + "speaker": "Nate", + "dia_id": "D5:14", + "text": "Pets really seem to do that to everyone don't they! So, what about your script now? Any ideas for the next steps?" + }, + { + "speaker": "Joanna", + "dia_id": "D5:15", + "text": "I've been doing my fair share of research and networking non-stop for it. It's tough, but I'm determined to make it happen." + }, + { + "speaker": "Nate", + "dia_id": "D5:16", + "text": "Great idea! that should hopefully get some more eyes on it. Keep up the hard work!" + }, + { + "speaker": "Joanna", + "dia_id": "D5:17", + "text": "Thanks so much, Nate! Your support means a lot. I'll keep working at it and hopefully the next steps will become clearer soon." + }, + { + "speaker": "Nate", + "dia_id": "D5:18", + "text": "Just make sure you don't quit - the path forward will show up soon. You got this!" + }, + { + "speaker": "Joanna", + "dia_id": "D5:19", + "text": "Appreciated! I think just having someone to support me throughout the whole process is such a blessing. It gives me the motivation to keep pushing forward." + }, + { + "speaker": "Nate", + "dia_id": "D5:20", + "text": "Glad to hear my support makes a difference, Joanna. I'm here for you!" + }, + { + "speaker": "Joanna", + "blip_caption": "a photo of a cinema ticket on a chair", + "dia_id": "D5:21", + "text": "Always good to hear! See you later!" + } + ], + "session_6_date_time": "1:43 pm on 24 March, 2022", + "session_6": [ + { + "speaker": "Nate", + "dia_id": "D6:1", + "text": "Hey Joanna! Long time no talk, how's it going? Crazy stuff's been happening since we last chatted." + }, + { + "speaker": "Joanna", + "dia_id": "D6:2", + "text": "Hey Nate! Been quite a ride - in a good way - had an audition yesterday for a writing gig." + }, + { + "speaker": "Nate", + "dia_id": "D6:3", + "text": "Congrats! How did it go? Are you excited?" + }, + { + "speaker": "Joanna", + "dia_id": "D6:4", + "text": "Thanks, Nate! It went alright. Mixed emotions - definitely excited but also a bit anxious. Keep those fingers crossed!" + }, + { + "speaker": "Nate", + "dia_id": "D6:5", + "text": "Yeah, I get it. Mixed emotions are rough, but I have faith in you! Keep me posted - you got this!" + }, + { + "speaker": "Joanna", + "dia_id": "D6:6", + "text": "Thanks, Nate! Your support means a lot. I'll make sure to keep you updated. Anything new on your end?" + }, + { + "speaker": "Nate", + "dia_id": "D6:7", + "text": "Yeah actually - I'm currently participating in the video game tournament again and it's INTENSE! There's so much adrenaline flowing." + }, + { + "speaker": "Joanna", + "img_url": [ + "https://blogger.googleusercontent.com/img/b/R29vZ2xl/AVvXsEjW88nSAhvDHq-ZyBSZlQ__czsDBW4hEFEvzdgkz06j4ByTnkAamu38jTEXQmzjkEgmS2VYyUYt4e0kz5cfhH7pChnTfAR2OKl9HZj7T91RxPZTV5qxTpi3jjpbFY1wD9Myi7633MAHlwiCL1LtCXt8iiQLed4Kzu2FY7thoxVxVBW8faEpQ-qyEFLwBTA/s4032/Photo%2023.07.23,%2014%2034%2052.jpg" + ], + "blip_caption": "a photo of a book shelf filled with books and magazines", + "query": "bookshelf writing books inspiration", + "dia_id": "D6:8", + "text": "Best of luck in the tournament! It sounds like it would be difficult to go through so many days of intense gaming! This is my go-to place for writing inspiration. It helps me stay sharp and motivated." + }, + { + "speaker": "Nate", + "dia_id": "D6:9", + "text": "Wow, that's a lot of books. Do you have any advice for someone like me who wants to pursue writing?" + }, + { + "speaker": "Joanna", + "dia_id": "D6:10", + "text": "Definitely! Read lots and try out different genres. Build a solid understanding of literature. Don't be afraid to write and share, even if it's just with friends. Practicing and gathering feedback will make you better. Have faith in yourself and continue following your writing dreams - it's tough but worth it." + }, + { + "speaker": "Nate", + "dia_id": "D6:11", + "text": "Thanks, Joanna. Really appreciate your help and kind words. I'm going to keep working hard on it and see what happens. Good luck with your project, I'm sure it will turn out great!" + }, + { + "speaker": "Joanna", + "dia_id": "D6:12", + "text": "Thanks Nate! Appreciate your kind words and support. Let's keep going for our dreams and work hard. Catch you later!" + }, + { + "speaker": "Nate", + "dia_id": "D6:13", + "text": "Bye Joanna! Take care!" + } + ], + "session_7_date_time": "7:37 pm on 15 April, 2022", + "session_7": [ + { + "speaker": "Nate", + "dia_id": "D7:1", + "text": "Hey Jo, guess what I did? Dyed my hair last week - come see!" + }, + { + "speaker": "Joanna", + "dia_id": "D7:2", + "text": "Wow, Nate! Can't wait to see it. Must feel so liberating! How're you feeling?" + }, + { + "speaker": "Nate", + "img_url": [ + "https://live.staticflickr.com/2378/2443069910_bf08328b2b_b.jpg" + ], + "blip_caption": "a photography of a man with purple hair and glasses taking a selfie", + "query": "purple hair selfie", + "dia_id": "D7:3", + "re-download": true, + "text": "I'm so stoked about it! Check it out!" + }, + { + "speaker": "Joanna", + "dia_id": "D7:4", + "text": "Wow, your new hair color looks amazing! What made you choose that shade? Tell me all about it!" + }, + { + "speaker": "Nate", + "dia_id": "D7:5", + "text": "Thanks Jo! I picked this color because it's bright and bold - like me! I wanted to stand out from the regular options." + }, + { + "speaker": "Joanna", + "img_url": [ + "https://i.redd.it/ruvtfm6ixchb1.jpg" + ], + "blip_caption": "a photo of a street with a stop sign and a cloudy sky", + "query": "sunset hike pink blue bold", + "dia_id": "D7:6", + "text": "That's amazing, Nate! Your boldness really inspired me. It reminded me of this gorgeous sunset I saw while hiking the other day. It made me realize the importance of showing the world who we are." + }, + { + "speaker": "Nate", + "dia_id": "D7:7", + "text": "Wow, that sunset looks awesome! Jealous! I bet you had a great time. Are there any more exciting trips coming up for you?" + }, + { + "speaker": "Joanna", + "dia_id": "D7:8", + "text": "I did! the sky was so gorgeous! Wish I had a vacation lined up, but right now my writing is consuming me. Hoping for some good news soon!" + }, + { + "speaker": "Nate", + "dia_id": "D7:9", + "text": "I understand, Joanna. Big projects can be so taxing. Keep me posted on how it goes, alright?" + }, + { + "speaker": "Joanna", + "dia_id": "D7:10", + "text": "Cheers, Nate! Your support means a lot. I'll definitely keep you updated." + }, + { + "speaker": "Nate", + "dia_id": "D7:11", + "text": "Sounds great, See you soon?" + }, + { + "speaker": "Joanna", + "dia_id": "D7:12", + "text": "Totally! Bye Nate!" + }, + { + "speaker": "Nate", + "dia_id": "D7:13", + "text": "Take care!" + } + ], + "session_8_date_time": "6:44 pm on 17 April, 2022", + "session_8": [ + { + "speaker": "Nate", + "dia_id": "D8:1", + "text": "Hey Joanna! Haven't talked with you in a while - how's it going?" + }, + { + "speaker": "Joanna", + "dia_id": "D8:2", + "text": "Hey Nate! Great to hear from you. I've been reading a lot in the past week! There's a lot of good books I forgot I owned." + }, + { + "speaker": "Nate", + "dia_id": "D8:3", + "text": "Sounds fun! I probably also have loads of books I haven't read in years. Sounds like a blast from the past!" + }, + { + "speaker": "Joanna", + "dia_id": "D8:4", + "text": "It really is! On a different note, I found an awesome hiking trail in my hometown yesterday! It was gorgeous. Nature is so inspiring, and it's a great way to reset. Do you know of any good hiking spots?" + }, + { + "speaker": "Nate", + "dia_id": "D8:5", + "text": "I'm not really into hiking but I'm curious to see what the trail looks like! I heard there's a nice trail just north of where I live." + }, + { + "speaker": "Joanna", + "dia_id": "D8:6", + "text": "Maybe I'll have to convince you to go with me one of these times!" + }, + { + "speaker": "Nate", + "dia_id": "D8:7", + "text": "Maybe! I do like nature, so that might be fun going with someone else." + }, + { + "speaker": "Joanna", + "dia_id": "D8:8", + "text": "Yeah, nature's awesome! I'm a huge fan of it, that's why I go!" + }, + { + "speaker": "Nate", + "dia_id": "D8:9", + "text": "Agreed, nature has a way of being so inspiring! I'm glad you found a way to reset and find peace in it." + }, + { + "speaker": "Joanna", + "dia_id": "D8:10", + "text": "Nature's always been my haven. Walking in it, feeling it, hearing the sounds - it's so calming. Worries and stress seem to vanish, and it's just me and the beauty around me." + }, + { + "speaker": "Nate", + "dia_id": "D8:11", + "text": "It's so crucial to find a little peace and remember life's beauty. For me, it's spending time with my pets and engaging in my hobbies; they let me take a break from reality. It's wild how small things can have such a powerful effect on our happiness, right?" + }, + { + "speaker": "Joanna", + "dia_id": "D8:12", + "text": "Yeah, Nate! Even the small things make life enjoyable and worth it. Taking time for your little friends and doing activities you love are like treasures that remind us how great and peaceful life is. We just gotta savor them!" + }, + { + "speaker": "Nate", + "img_url": [ + "https://i.redd.it/3yrgb86abnl81.jpg" + ], + "blip_caption": "a photo of a turtle and a strawberry in a bowl", + "query": "pets turtles playing", + "dia_id": "D8:13", + "text": "Speaking of which, here they go again!" + }, + { + "speaker": "Joanna", + "dia_id": "D8:14", + "text": "So cute! I love your turtles so much!" + }, + { + "speaker": "Nate", + "img_url": [ + "https://beadyarnspatula.files.wordpress.com/2023/05/img_6671.jpg" + ], + "blip_caption": "a photo of a bowl of ice cream and a bowl of sprinkles", + "query": "coconut milk ice cream red sprinkles bowl", + "dia_id": "D8:15", + "text": "Me too! I love watching them play to simply enjoy the peaceful moments of life. Sometimes I even bring them in the kitchen so they can watch me make food like this!" + }, + { + "speaker": "Joanna", + "dia_id": "D8:16", + "text": "I love your icecream so much! I wish I could make it the way you do!" + }, + { + "speaker": "Nate", + "dia_id": "D8:17", + "text": "Thanks! It's dairy-free and so easy. Wanna get the recipe?" + }, + { + "speaker": "Joanna", + "dia_id": "D8:18", + "text": "Sure! I'm lactose intolerant, so I'll just need the dairy-free recipe! " + }, + { + "speaker": "Nate", + "dia_id": "D8:19", + "text": "No prob. I made it with coconut milk, vanilla extract, sugar, and a pinch of salt. After chilling it in the fridge, I put it in the ice cream maker and froze it until it was scoopable." + }, + { + "speaker": "Joanna", + "dia_id": "D8:20", + "text": "Wow, sounds delicious! I'm going to try making it tonight! Thank you for sharing the recipe!" + }, + { + "speaker": "Nate", + "dia_id": "D8:21", + "text": "Hey Joanna, glad I could help. Let me know how it turns out!" + }, + { + "speaker": "Joanna", + "dia_id": "D8:22", + "text": "Got it, Nate. I'll definitely let you know how it turns out. Thanks for sharing the recipe!" + } + ], + "session_9_date_time": "7:44 pm on 21 April, 2022", + "session_9": [ + { + "speaker": "Joanna", + "img_url": [ + "https://threeteacherstalk.files.wordpress.com/2021/06/fnqw-6.18.jpg" + ], + "blip_caption": "a photo of a notebook with a notepad and a piece of paper", + "query": "writers group notebook notes ideas storytelling", + "dia_id": "D9:1", + "text": "Hey Nate! Long time no talk! I wanted to tell ya I just joined a writers group. It's unbelievable--such inspirational people who really get my writing. I'm feeling so motivated and supported, it's like I finally belong somewhere!" + }, + { + "speaker": "Nate", + "blip_caption": "a photo of a cup of ice cream with a cherry on top", + "dia_id": "D9:2", + "text": "Hey Joanna! That's awesome! Having a supportive group around you can really make a difference. What kind of projects are you working on with them?" + }, + { + "speaker": "Joanna", + "dia_id": "D9:3", + "text": "Thanks, Nate! We've made some great progress. I'm working on one with my group called \"Finding Home.\" It's a script about a girl on a journey to find her true home. I find it really rewarding and emotional. What about you? Any upcoming gaming tournaments?" + }, + { + "speaker": "Nate", + "dia_id": "D9:4", + "text": "Hi Joanna! \"Finding Home\" sounds really special. Must be so meaningful to work on. I've got a gaming tournament next month and I'm feeling good about it. It's gonna be my 4th one!" + }, + { + "speaker": "Joanna", + "img_url": [ + "https://get.pxhere.com/photo/black-and-white-old-audience-fashion-performance-art-theatre-children-stage-performance-entertainment-scenario-performing-arts-monochrome-photography-musical-theatre-541499.jpg" + ], + "blip_caption": "a photography of a man in a striped suit is performing on stage", + "query": "theater stage", + "dia_id": "D9:5", + "re-download": true, + "text": "Yeah, I bet the nerves and excitement are quite a rush! I remember when I did my first play, I was so nervous I forgot my lines. It was embarrassing, but it taught me how important it is to prepare and stay in the moment." + }, + { + "speaker": "Nate", + "dia_id": "D9:6", + "text": "Sounds like you had an interesting time on stage! It's always a learning experience. Have you ever considered going back to acting? Is that you in the photo?" + }, + { + "speaker": "Joanna", + "dia_id": "D9:7", + "text": "Yeah, that's me in that photo! Acting was my first passion, but now I really shine in writing. It helps me express myself in a new way, but who knows, maybe I'll go back to acting someday. Never say never!" + }, + { + "speaker": "Nate", + "blip_caption": "a photo of a turtle laying on a bed of rocks and gravel", + "dia_id": "D9:8", + "text": "Go for it! Follow your passion for writing, but if acting really makes you happy, give it a shot as well. Who knows what'll happen! Any particular movies that spark your writing?" + }, + { + "speaker": "Joanna", + "dia_id": "D9:9", + "text": "Thanks Nate! I'm gonna keep writing, but if acting calls out I might give it a try. I really enjoy dramas and emotionally-driven films. What about you? What inspires your passion?" + }, + { + "speaker": "Nate", + "img_url": [ + "https://www.trustedreviews.com/wp-content/uploads/sites/54/2020/10/20201026_152436-scaled-e1604409626602.jpg" + ], + "blip_caption": "a photography of a black xbox controller sitting on top of a wooden table", + "query": "video game controller", + "dia_id": "D9:10", + "re-download": true, + "text": "I love fantasy and sci-fi movies, they're a great escape and get my imagination going. Playing video games is a great way to express my creativity and passion." + }, + { + "speaker": "Joanna", + "dia_id": "D9:11", + "text": "That's awesome! I love how video games can really spark your imagination. Do you have a favorite fantasy or sci-fi movie?" + }, + { + "speaker": "Nate", + "img_url": [ + "https://i.redd.it/7vtqewbtg1181.jpg" + ], + "blip_caption": "a photo of a shelf with a lot of books on it", + "query": "lord of the rings trilogy dvd boxset", + "dia_id": "D9:12", + "text": "Yeah, for sure! This trilogy is one of my faves. The world building, battles, and storytelling always blow me away!" + }, + { + "speaker": "Joanna", + "dia_id": "D9:13", + "text": "Wow, that's great to hear! What books do you enjoy? I'm always up for some new book recommendations." + }, + { + "speaker": "Nate", + "img_url": [ + "https://i.redd.it/o2ifb25e7qa31.jpg" + ], + "blip_caption": "a photo of a bunch of books on a table", + "query": "fantasy novels dragon cover series", + "dia_id": "D9:14", + "text": "I love this series. It has adventures, magic, and great characters - it's a must-read!" + }, + { + "speaker": "Joanna", + "dia_id": "D9:15", + "text": "Heard of that series! It's been on my list forever. Thanks for the recommendation, Nate. I'm definitely going to check it out!" + }, + { + "speaker": "Nate", + "dia_id": "D9:16", + "text": "No problem, glad to see an interest. Let me know what you think when you check it out." + }, + { + "speaker": "Joanna", + "dia_id": "D9:17", + "text": "Thanks Nate! I'll definitely let you know my thoughts. Take care and have a great day!" + }, + { + "speaker": "Nate", + "dia_id": "D9:18", + "text": "See you! Good chatting with you! Have a great day!" + } + ], + "session_10_date_time": "11:54 am on 2 May, 2022", + "session_10": [ + { + "speaker": "Joanna", + "blip_caption": "a photo of a person holding a book openhemer", + "dia_id": "D10:1", + "text": "Hey Nate, how's it going? I took your reccomendation and watched \"The Lord of the Rings\" Trilogy last night! It was awesome!" + }, + { + "speaker": "Nate", + "img_url": [ + "https://i.pinimg.com/originals/23/13/24/231324d44f9471ead5535950153378f1.jpg" + ], + "blip_caption": "a photo of a gaming room with a computer and a gaming chair", + "query": "gaming setup", + "dia_id": "D10:2", + "text": "Glad to hear you enjoyed it! It's probably the greatest trilogy of all time! As for me, life's been ok, just taking care of this." + }, + { + "speaker": "Joanna", + "dia_id": "D10:3", + "text": "Wow, Nate! I'm proud of what you did. Your gaming room looks great - have you been gaming a lot recently?" + }, + { + "speaker": "Nate", + "blip_caption": "a photo of a woman with purple hair and a black dress", + "dia_id": "D10:4", + "text": "Gaming has been my focus - practicing a lot and even winning a few tournaments. Last week I won my second tournament!" + }, + { + "speaker": "Joanna", + "dia_id": "D10:5", + "text": "Wow, congrats! What game were you playing?" + }, + { + "speaker": "Nate", + "dia_id": "D10:6", + "text": "Thanks! I usually play CS:GO, but I tried my hand at the local Street Fighter tournament this time since I play that game a lot with my friends, and turns out I'm really good!" + }, + { + "speaker": "Joanna", + "dia_id": "D10:7", + "text": "Nice! That must have been a surprise. How did it feel to finally win one?" + }, + { + "speaker": "Nate", + "dia_id": "D10:8", + "text": "It was super awesome! So much adrenaline went into that last match, and the other finalist even shook my hand! Enough about me though, how about you? What have you been up to?" + }, + { + "speaker": "Joanna", + "img_url": [ + "http://ventitobakery.com/cdn/shop/products/white-gluten-dairy-free-cake.jpg" + ], + "blip_caption": "a photo of a cake with white frosting on a wooden table", + "query": "dairy-free cake decorated", + "dia_id": "D10:9", + "text": "Not much is new other than the screenplay. Been working on some projects and testing out dairy-free dessert recipes for friends and fam. Here's a pic of a cake I made recently!" + }, + { + "speaker": "Nate", + "dia_id": "D10:10", + "text": "That looks really good! I love the way the frosting turned out!" + }, + { + "speaker": "Joanna", + "dia_id": "D10:11", + "text": "Thanks! It's dairy-free vanilla with strawberry filling and coconut cream frosting. I gotta say, I really like your coconut reccomendation you gave a while back!" + }, + { + "speaker": "Nate", + "dia_id": "D10:12", + "text": "Wow, Joanna, that looks amazing! I bet it tastes great - you're so talented at making dairy-free desserts!" + }, + { + "speaker": "Joanna", + "dia_id": "D10:13", + "text": "Thanks Nate! I really appreciate it. I love experimenting in the kitchen, coming up with something tasty. Cooking and baking are my creative outlets. Especially when I'm snackin' dairy-free, trying to make the desserts just as delicious - it's a rewarding challenge! Seeing the smiles on everyone's faces when they try it - it's a total win!" + }, + { + "speaker": "Nate", + "dia_id": "D10:14", + "text": "That's great, Joanna! It must be so rewarding to see everyone enjoying your creations. Keep up the good work!" + }, + { + "speaker": "Joanna", + "dia_id": "D10:15", + "text": "Thanks, Nate! Appreciate all the help. Gonna keep trying new things. See ya later!" + }, + { + "speaker": "Nate", + "dia_id": "D10:16", + "text": "Bye!" + } + ], + "session_11_date_time": "3:35 pm on 12 May, 2022", + "session_11": [ + { + "speaker": "Joanna", + "dia_id": "D11:1", + "text": "Hey Nate! Great to hear from you! Quite a week since we last talked - something awesome happened to me!" + }, + { + "speaker": "Nate", + "dia_id": "D11:2", + "text": "Hey Jo! Great hearing from you! What happened?" + }, + { + "speaker": "Joanna", + "dia_id": "D11:3", + "text": "I went hiking and found some more amazing trails in my town. It was such an awesome experience! I think I am an expert hiker now." + }, + { + "speaker": "Nate", + "dia_id": "D11:4", + "text": "Sounds great, Jo! Happy you had an awesome experience. Did you happen to take any photos of it?" + }, + { + "speaker": "Joanna", + "img_url": [ + "https://burst.shopifycdn.com/photos/large-rolling-waterfall-though-a-lush-green-hill.jpg" + ], + "blip_caption": "a photo of a waterfall with a dark sky in the background", + "query": "waterfall lush greenery", + "dia_id": "D11:5", + "text": "Yeah, I did! Loved this spot on the hike. The rush of the water was so soothing." + }, + { + "speaker": "Nate", + "dia_id": "D11:6", + "text": "Wow, looks great! Where did you take this picture? I love the dark sky and green scenery." + }, + { + "speaker": "Joanna", + "dia_id": "D11:7", + "text": "Thanks! I took this photo at a beautiful location called Whispering Falls. It was really peaceful and serene." + }, + { + "speaker": "Nate", + "dia_id": "D11:8", + "text": "I wish I could have been there! Your hikes sound like a blast." + }, + { + "speaker": "Joanna", + "dia_id": "D11:9", + "text": "It was awesome, Nate. The sound of that place and the beauty of nature made me so calm and peaceful. Everything else faded away and all that mattered was the present." + }, + { + "speaker": "Nate", + "dia_id": "D11:10", + "text": "That's great. Glad you found a spot that calms you down - nature sure can be a break from the craziness." + }, + { + "speaker": "Joanna", + "dia_id": "D11:11", + "text": "Nature totally inspires me and it's so calming to be surrounded by its beauty. Hiking has opened up a whole new world for me and I feel like a different person now." + }, + { + "speaker": "Nate", + "dia_id": "D11:12", + "text": "Wow, Jo, that's really cool! It's great to have something that gets those creative juices flowing." + }, + { + "speaker": "Joanna", + "dia_id": "D11:13", + "text": "I always feel like I could write a whole movie when I'm out there in cool places like that!" + }, + { + "speaker": "Nate", + "dia_id": "D11:14", + "text": "Wow! That's really cool that it inspires you that much! For me I just get deep in thought and think about my life or new recipes." + }, + { + "speaker": "Joanna", + "dia_id": "D11:15", + "text": "I think about my life too sometimes when I'm out and about, but there was something special about these trails that made me feel like writing a drama." + }, + { + "speaker": "Nate", + "dia_id": "D11:16", + "text": "Hey, we should go together sometime, don't you think? Maybe I'll start to think of a drama myself and publish my own screenplay." + }, + { + "speaker": "Joanna", + "dia_id": "D11:17", + "text": "Haha, now that would be something! Sure, you should come down and join me on the trails sometime!" + }, + { + "speaker": "Nate", + "dia_id": "D11:18", + "text": "Sounds like a plan! Thanks for the invite Joanna!" + }, + { + "speaker": "Joanna", + "dia_id": "D11:19", + "text": "Sure thing Nate! See you later!" + }, + { + "speaker": "Nate", + "dia_id": "D11:20", + "text": "See ya!" + } + ], + "session_12_date_time": "7:49 pm on 20 May, 2022", + "session_12": [ + { + "speaker": "Nate", + "dia_id": "D12:1", + "text": "Hey Joanna! How've you been? Been a busy week since we talked." + }, + { + "speaker": "Joanna", + "dia_id": "D12:2", + "text": "Hey Nate! Just finished something - pretty wild journey!" + }, + { + "speaker": "Nate", + "img_url": [ + "https://i.redd.it/rgx6yqabu5e91.jpg" + ], + "blip_caption": "a photo of a dog laying on a couch in a living room", + "query": "cute dog max", + "dia_id": "D12:3", + "text": "Way to go! I just got a new addition to the family, this is Max!" + }, + { + "speaker": "Joanna", + "dia_id": "D12:4", + "text": "Wow, he's adorable! How long have you had him? I can see why you're thrilled!" + }, + { + "speaker": "Nate", + "dia_id": "D12:5", + "text": "Thanks! It's awesome - he's adopted and so full of energy, and he's filling my life with so much joy. He's even keeping my other pets active." + }, + { + "speaker": "Joanna", + "dia_id": "D12:6", + "text": "Pets sure do make life better! Glad Max is bringing you lots of joy." + }, + { + "speaker": "Nate", + "dia_id": "D12:7", + "text": "Yep, totally! Pets make us so much happier and never let us down. Have you thought any more of getting one of your own?" + }, + { + "speaker": "Joanna", + "dia_id": "D12:8", + "text": "Unfortunately, allergies make it so I don't really want to get any, and I'm too lazy to research alternative pets for my allergies." + }, + { + "speaker": "Nate", + "dia_id": "D12:9", + "text": "Aww, that's unfortunate. It's nice seeing the joy pets bring to others, though. How do you find comfort when you don't have any?" + }, + { + "speaker": "Joanna", + "dia_id": "D12:10", + "text": "Writing and creative projects are what get me through tough times. I'm also grateful for my supportive friends." + }, + { + "speaker": "Nate", + "dia_id": "D12:11", + "text": "Wow, that's awesome! Those both can definitely be therapeutic. It's great to have such positive relationships that make such a great impact." + }, + { + "speaker": "Joanna", + "img_url": [ + "https://i.redd.it/0hvywunpjfs91.jpg" + ], + "blip_caption": "a photo of a notepad with a dog on it and a pen", + "query": "handwritten screenplay notebook.", + "dia_id": "D12:12", + "text": "Yeah. It's so nice to have friends who understand and appreciate my work - it's priceless being able to talk about it together and receive feedback. Here's a look at what I've been working on – it's been quite a journey, but I made it!" + }, + { + "speaker": "Nate", + "dia_id": "D12:13", + "text": "Wow, that looks great Joanna! Is that your third one?" + }, + { + "speaker": "Joanna", + "dia_id": "D12:14", + "text": "Yep! I chose to write about this because it's really personal. It's about loss, identity, and connection. It's a story I've had for ages but just got the guts to write it. It was hard, but I'm so proud of it." + }, + { + "speaker": "Nate", + "dia_id": "D12:15", + "text": "That sounds impressive. You really do like writing about sadness and loss don't you." + }, + { + "speaker": "Joanna", + "dia_id": "D12:16", + "text": "Thanks, Nate! Yeah I really do. I had to be vulnerable and dig deep into those topics. But I think meaningful stories come from personal experiences and feelings. It was scary, but I found that I write best when I'm being true to myself - even if it's hard." + }, + { + "speaker": "Nate", + "dia_id": "D12:17", + "text": "Well done, Joanna! It takes guts to explore your experiences and feelings. I'm proud of you for staying strong and being true to yourself. Keep it up!" + }, + { + "speaker": "Joanna", + "dia_id": "D12:18", + "text": "Thanks, Nate! Your support really means a lot. Knowing I've got people like you cheering me on makes this journey way easier." + }, + { + "speaker": "Nate", + "dia_id": "D12:19", + "text": "No worries, Joanna! Keep going and reach for your dreams. You've got tons of talent and potential!" + } + ], + "session_13_date_time": "3:00 pm on 25 May, 2022", + "session_13": [ + { + "speaker": "Nate", + "dia_id": "D13:1", + "text": "Hey Jo! Been ages since we last talked. Here's something cool that happened the other day - I took Max for a walk and ran into this super nice couple who had a dog. It turns out they live close by. We decided to do doggy playdates, which is awesome considering we all need friends for our pets." + }, + { + "speaker": "Joanna", + "dia_id": "D13:2", + "text": "Hey Nate! Great to hear from you. Sounds like a nice encounter on your walk. Connecting with others who have pets can be uplifting and rewarding." + }, + { + "speaker": "Nate", + "dia_id": "D13:3", + "text": "It's like fate. Having a walking buddy forMax will be great. He really likes the other dog too!" + }, + { + "speaker": "Joanna", + "dia_id": "D13:4", + "text": "Awesome! Did you get to know the couple very well? What were they like?" + }, + { + "speaker": "Nate", + "dia_id": "D13:5", + "text": "They actually didn't share a whole lot in common with me besides the love of animals, but I think that was all we needed to share in common to be good friends!" + }, + { + "speaker": "Joanna", + "dia_id": "D13:6", + "text": "That's really cool that you can just go out and meet people like that, keep it up Nate!" + }, + { + "speaker": "Nate", + "dia_id": "D13:7", + "text": "Thanks! I just really enjoy watching our pets play with one another. Its like a dream come true seeing my dog so happy." + }, + { + "speaker": "Joanna", + "dia_id": "D13:8", + "text": "I can see why having a peaceful presence around could help relieve stress. Having someone or something to come home to for a sense of calm would be helpful for relaxation." + }, + { + "speaker": "Nate", + "img_url": [ + "https://i.redd.it/usll6z99c2tb1.jpg" + ], + "blip_caption": "a photo of a stuffed animal laying on a bed", + "query": "dog cozy blanket toy", + "dia_id": "D13:9", + "text": "Yep, Joanna. It's great! Looky here, I got this new pup for you!" + }, + { + "speaker": "Joanna", + "dia_id": "D13:10", + "text": "Awww! It's so cute! I love the thought Nate!" + }, + { + "speaker": "Nate", + "dia_id": "D13:11", + "text": "Thanks! It's a stuffed animal to remind you of the good vibes." + }, + { + "speaker": "Joanna", + "dia_id": "D13:12", + "text": "That's so sweet! I'll cherish that little guy with all my heart!" + }, + { + "speaker": "Nate", + "dia_id": "D13:13", + "text": "Yeah! It's like having joy in your pocket. It always makes me grin when I look at it." + }, + { + "speaker": "Joanna", + "dia_id": "D13:14", + "text": "That's great, Nate! Appreciate the small joys like that cute stuffed animal. It's a nice reminder!" + }, + { + "speaker": "Nate", + "dia_id": "D13:15", + "text": "Agreed, those little things sure do make life better!" + }, + { + "speaker": "Joanna", + "dia_id": "D13:16", + "text": "I'll always remember those moments that bring us happiness and remind us that life is great!" + }, + { + "speaker": "Nate", + "dia_id": "D13:17", + "text": "Sure, Joanna! It's all about finding those little things and cherishing them, otherwise it's easy to get down!" + }, + { + "speaker": "Joanna", + "dia_id": "D13:18", + "text": "Thinking back to the tough times finishing my screenplay made me realize it's those moments that bring joy and make the journey worth it." + }, + { + "speaker": "Nate", + "dia_id": "D13:19", + "text": "Yeah, those little moments make it all worth it, especially during tough times. Enjoying the ride is key." + }, + { + "speaker": "Joanna", + "dia_id": "D13:20", + "text": "Appreciating the journey and being aware of those happy moments can be a game-changer! It keeps us focused on our dreams. Can't wait to show it to you. I value your opinion!" + }, + { + "speaker": "Nate", + "dia_id": "D13:21", + "text": "Can't wait to see it, Joanna! I'm here to support you." + }, + { + "speaker": "Joanna", + "dia_id": "D13:22", + "text": "Thanks, Nate! Your support is greatly appreciated. I'll make sure to keep you updated." + }, + { + "speaker": "Nate", + "dia_id": "D13:23", + "text": "No worries! You've got this. Keep it up!" + } + ], + "session_14_date_time": "5:44 pm on 3 June, 2022", + "session_14": [ + { + "speaker": "Joanna", + "dia_id": "D14:1", + "text": "Nate, after finishing my screenplay I got a rejection letter from a major company. It really bummed me out." + }, + { + "speaker": "Nate", + "dia_id": "D14:2", + "text": "Sorry to hear that, Joanna. Rejection stinks, but it doesn't mean you're not talented. Don't give up on your dreams!" + }, + { + "speaker": "Joanna", + "dia_id": "D14:3", + "text": "Thanks, Nate. It can feel like a step back sometimes. But I appreciate your kind words and encouragement." + }, + { + "speaker": "Nate", + "dia_id": "D14:4", + "text": "Sure, just make sure you keep going and believing in yourself. Did something happen with the company?" + }, + { + "speaker": "Joanna", + "dia_id": "D14:5", + "text": "They just sent me a generic rejection letter without much feedback. It's disheartening not knowing why it didn't work out." + }, + { + "speaker": "Nate", + "dia_id": "D14:6", + "text": "Ugh, that's so frustrating. But don't get discouraged, just keep going." + }, + { + "speaker": "Joanna", + "dia_id": "D14:7", + "text": "Yeah, you're right. I won't let this bring me down. Thanks for your support. What have you been up to lately?" + }, + { + "speaker": "Nate", + "dia_id": "D14:8", + "text": "I've been doing great - I just won another regional video game tournament last week! It was so cool, plus I met some new people. Connecting with fellow gamers is always awesome." + }, + { + "speaker": "Joanna", + "dia_id": "D14:9", + "text": "Way to go, Nate! Congratulations on your victory in the tournament! It must feel great to be recognized for your gaming skills." + }, + { + "speaker": "Nate", + "dia_id": "D14:10", + "text": "Thanks, Joanna! Winning was a huge confidence boost and shows my hard work paid off. I'm really happy with my progress." + }, + { + "speaker": "Joanna", + "dia_id": "D14:11", + "text": "I am as well! It's great to hear from you about your tournaments throughout the years!" + }, + { + "speaker": "Nate", + "dia_id": "D14:12", + "text": "Thanks! I has been a while since my first tournament hasn't it? I appreciate your support!" + }, + { + "speaker": "Joanna", + "dia_id": "D14:13", + "text": "Anytime Nate! I'm here for you every step of the way." + }, + { + "speaker": "Nate", + "blip_caption": "a photo of a purple and blue controller with a star field design", + "dia_id": "D14:14", + "text": "I talked to some of the guys at the tournament afterwards, and they said they wanted to hang out later!" + }, + { + "speaker": "Joanna", + "dia_id": "D14:15", + "text": "Sounds like fun! It's good to have friends that share your interests!" + }, + { + "speaker": "Nate", + "dia_id": "D14:16", + "text": "For sure! They asked for some tips in how to improve their game, so I said I could help." + }, + { + "speaker": "Joanna", + "dia_id": "D14:17", + "text": "Good on you for helping strangers out! Stepping outside your comfort zone is always great." + }, + { + "speaker": "Nate", + "dia_id": "D14:18", + "text": "Thanks, I just like helping people. Do you have any plans for the weekend?" + }, + { + "speaker": "Joanna", + "dia_id": "D14:19", + "text": "Yep, I'm hiking with some buddies this weekend. We're checking out a new trail with a rad waterfall. Can't wait! Do you have any fun plans?" + }, + { + "speaker": "Nate", + "dia_id": "D14:20", + "text": "Sounds great! Have fun with that. I'm organizing a gaming party two weekends later - it'll be hectic but fun!" + }, + { + "speaker": "Joanna", + "dia_id": "D14:21", + "text": "Oh? Are you going to invite your tournament friends?" + }, + { + "speaker": "Nate", + "dia_id": "D14:22", + "text": "Definitely! And some old friends and teamates from other tournaments." + }, + { + "speaker": "Joanna", + "dia_id": "D14:23", + "text": "Sounds like fun, Nate! I wish you the best on your party. Have a blast!" + }, + { + "speaker": "Nate", + "dia_id": "D14:24", + "text": "Thanks Joanna! I'm sure it'll be a blast. I'm even getting everyone custom controller decorations just for coming!" + }, + { + "speaker": "Joanna", + "dia_id": "D14:25", + "text": "Wow, I bet they'll love that! What a sweet idea." + }, + { + "speaker": "Nate", + "dia_id": "D14:26", + "text": "I know right? Have a great hike. Take lots of pics! See ya later!" + }, + { + "speaker": "Joanna", + "dia_id": "D14:27", + "text": "Thanks Nate! See you later!" + } + ], + "session_15_date_time": "2:12 pm on 5 June, 2022", + "session_15": [ + { + "speaker": "Joanna", + "blip_caption": "a photo of a spider - man poster hanging on a wall", + "dia_id": "D15:1", + "text": "Hey Nate! Yesterday was crazy cool - I wrote a few bits for a screenplay that appeared on the big screen yesterday! It was nerve-wracking but so inspiring to see my words come alive!" + }, + { + "speaker": "Nate", + "dia_id": "D15:2", + "text": "Congrats, Joanna! Seeing your hard work pay off like that must've felt amazing. I bet it was scary too, but awesome! You're so inspiring. By the way, last time we saw eachother, I noticed a spiderman pin on your purse. Is Spider-Man your favorite superhero, or do you have another fave?" + }, + { + "speaker": "Joanna", + "dia_id": "D15:3", + "text": "Thanks, Nate! It was a real roller coaster, but seeing the hard work pay off was amazing. Spider-Man has always been a favorite of mine - I mean, who doesn't love Peter Parker's struggles between being a hero and being a person? But I'm kind of a sucker for any superhero - everyone has their own rad story and powers. Do you have a favorite superhero?" + }, + { + "speaker": "Nate", + "img_url": [ + "https://live.staticflickr.com/7148/6768759075_81b67d203f_b.jpg" + ], + "blip_caption": "a photography of a toy iron man standing on a white surface", + "query": "iron man action figures", + "dia_id": "D15:4", + "re-download": true, + "text": "That's great, Joanna! Iron Man is my top pick. I love his tech and that sarcastic humor. Seeing these figures just makes me feel invincible!" + }, + { + "speaker": "Joanna", + "dia_id": "D15:5", + "text": "Wow, Nate! That's awesome. I love the tech and funny jokes of Iron Man too. What made you get that figure?" + }, + { + "speaker": "Nate", + "dia_id": "D15:6", + "text": "Thanks Joanna! I got it because it reminded me of something I love. Its presence in my room is a good reminder to keep working on my goals. Any inspiring things in your room?" + }, + { + "speaker": "Joanna", + "dia_id": "D15:7", + "text": "My cork board is full of inspiring quotes and pictures for motivation and creativity. It's my little corner of inspiration." + }, + { + "speaker": "Nate", + "dia_id": "D15:8", + "text": "Wow Joanna, that sounds great! Could you show me a picture of it?" + }, + { + "speaker": "Joanna", + "img_url": [ + "https://i.pinimg.com/originals/cc/b0/14/ccb0147bd55dfa436f284fb5939a7808.jpg" + ], + "blip_caption": "a photo of a picture frame with a picture of a family", + "query": "cork board quotes photos loved ones mementos", + "dia_id": "D15:9", + "text": "Here ya go, a pic of my cork board. It's got quotes, photos, and little keepsakes." + }, + { + "speaker": "Nate", + "dia_id": "D15:10", + "text": "That's a great pic of your family! What made you hang it on your cork board?" + }, + { + "speaker": "Joanna", + "dia_id": "D15:11", + "text": "Thanks, Nate! Having that picture on my cork board reminds me of the love and encouragement from them every day." + }, + { + "speaker": "Nate", + "dia_id": "D15:12", + "text": "That's great, Joanna. Family support is invaluable. It's so good to have those reminders." + }, + { + "speaker": "Joanna", + "dia_id": "D15:13", + "text": "Absolutely, it means a lot and keeps me going." + }, + { + "speaker": "Nate", + "dia_id": "D15:14", + "text": "I really should start a cork board of my own shouldn't I. That seems like a really valuable thing!" + }, + { + "speaker": "Joanna", + "dia_id": "D15:15", + "text": "I would definitely recommend it! As long as your willing to explain what it is to your friends." + }, + { + "speaker": "Nate", + "dia_id": "D15:16", + "text": "Of course! Well see you later Joanna!" + }, + { + "speaker": "Joanna", + "dia_id": "D15:17", + "text": "Bye Nate!" + } + ], + "session_16_date_time": "10:55 am on 24 June, 2022", + "session_16": [ + { + "speaker": "Joanna", + "dia_id": "D16:1", + "text": "Hey Nate, long time no see! How have you been? I just got done submitting my recent screenplay to a film contest just to see how others might like it!" + }, + { + "speaker": "Nate", + "dia_id": "D16:2", + "text": "That's really cool Joanna! I hope it does well, and I've been doing great! The gaming party was a great success! We even played some Chess afterward just for fun." + }, + { + "speaker": "Joanna", + "dia_id": "D16:3", + "text": "Nice! Did your friends like the controller accessories?" + }, + { + "speaker": "Nate", + "dia_id": "D16:4", + "text": "Most of them did! I can't say if all of them will continue to use them or not, but that's beside the point." + }, + { + "speaker": "Joanna", + "dia_id": "D16:5", + "text": "Yeah Nate, you're right. It doesn't matter if they use it, its the thought that matters right?" + }, + { + "speaker": "Nate", + "dia_id": "D16:6", + "text": "Absolutely! There were 7 people that attended, and 6 of them said they'd want to do it again next month!" + }, + { + "speaker": "Joanna", + "dia_id": "D16:7", + "text": "That sounds like a huge success then! Congrats!" + }, + { + "speaker": "Nate", + "img_url": [ + "https://www.stockvault.net/data/2022/06/05/293150/preview16.jpg" + ], + "blip_caption": "a photography of a group of people sitting on a bench", + "query": "vegan coconut milk ice cream vegan diet group", + "dia_id": "D16:8", + "re-download": true, + "text": "Thanks! On another note, I made vegan ice cream last Friday and shared it with some people in my vegan diet group. It was awesome!" + }, + { + "speaker": "Joanna", + "dia_id": "D16:9", + "text": "Mm, yum! Can you give me the recipe for that? It sounds like it'd be a good recipe!" + }, + { + "speaker": "Nate", + "dia_id": "D16:10", + "text": "Sure thing! I can give it to you tomorrow, how does that sound?" + }, + { + "speaker": "Joanna", + "dia_id": "D16:11", + "text": "Awesome! I'm going to make it for my family this weekend - can't wait!" + }, + { + "speaker": "Nate", + "dia_id": "D16:12", + "text": "Nice one, Joanna! Hope you and your family like it. Let me know how it went!" + }, + { + "speaker": "Joanna", + "dia_id": "D16:13", + "text": "Sure thing! They love it when I make them new things!" + }, + { + "speaker": "Nate", + "dia_id": "D16:14", + "text": "Then I have no doubt they'll love the icecream!" + }, + { + "speaker": "Joanna", + "dia_id": "D16:15", + "text": "Thanks Nate! Your support is greatly appreciated. I'll make sure to keep you updated." + }, + { + "speaker": "Nate", + "dia_id": "D16:16", + "text": "Can't wait to hear about it. Have a great day! Take care." + } + ], + "session_17_date_time": "2:34 pm on 10 July, 2022", + "session_17": [ + { + "speaker": "Nate", + "img_url": [ + "https://i.redd.it/p59z4xlbytfa1.jpg" + ], + "blip_caption": "a photo of a television screen showing a trophy and a trophy", + "query": "video game tournament trophy game screenshot", + "dia_id": "D17:1", + "text": "Hey Joanna, check this out! I won my fourth video game tournament on Friday! It was awesome competing and showing off my skills - and the victory was indescribable. I'm really proud that I can make money doing what I love. This one was online!" + }, + { + "speaker": "Joanna", + "dia_id": "D17:2", + "text": "Congrats, Nate! That's awesome! So proud of you. Your hard work really paid off - keep it up! BTW, I took a road trip for research for my next movie while you were winning. Much-needed break and a chance to explore new places and get inspired." + }, + { + "speaker": "Nate", + "dia_id": "D17:3", + "text": "Thanks, Joanna! Your support means a lot to me. That road trip sounds great! Where did you go? Did you discover any interesting places?" + }, + { + "speaker": "Joanna", + "img_url": [ + "https://upload.wikimedia.org/wikipedia/commons/9/91/Old_Geodesy_library_books.jpg" + ], + "blip_caption": "a photography of a book shelf filled with lots of books", + "query": "vintage library woodhaven midwest", + "dia_id": "D17:4", + "re-download": true, + "text": "Thanks Nate! Appreciate your kind words. I went to Woodhaven, a small town in the Midwest. Got to see some lovely scenery and historic buildings. Checked out the library there, it had a cool old book collection!" + }, + { + "speaker": "Nate", + "dia_id": "D17:5", + "text": "That place looks interesting! Did you find any cool books there?" + }, + { + "speaker": "Joanna", + "img_url": [ + "https://i.redd.it/sqyihv2m1j871.jpg" + ], + "blip_caption": "a photo of a book with writing on it and a pen", + "query": "old journal stories sketches handwritten pages", + "dia_id": "D17:6", + "text": "I stumbled upon this super cool book from the 1900s with stories and sketches - so awesome to read about the town and the people living there!" + }, + { + "speaker": "Nate", + "dia_id": "D17:7", + "text": "That sounds really interesting! Anyting specific stick out to you about it?" + }, + { + "speaker": "Joanna", + "dia_id": "D17:8", + "text": "Woodhaven has had an interesting past with lots of cool people. Seeing how much it changed sparked ideas for my next script." + }, + { + "speaker": "Nate", + "dia_id": "D17:9", + "text": "Real-life stories are the best for inspiration. Can't wait to hear about your next one. Keep it up!" + }, + { + "speaker": "Joanna", + "dia_id": "D17:10", + "text": "Thanks, Nate! I'm stoked about this new script. It's different from my previous work, but it has the potential to be something awesome! I'll be sure to keep you posted." + }, + { + "speaker": "Nate", + "dia_id": "D17:11", + "text": "I'm sure it will do just as well as your last one! Keep on trying and believe in yourself!" + }, + { + "speaker": "Joanna", + "dia_id": "D17:12", + "text": "Thanks, Nate! Your encouragement really means a lot to me. You're the best for supporting me in my writing journey." + }, + { + "speaker": "Nate", + "dia_id": "D17:13", + "text": "I'm always here for you! You've got so much talent, just keep going for it!" + }, + { + "speaker": "Joanna", + "img_url": [ + "https://i.imgur.com/bJiLDHb.jpg" + ], + "blip_caption": "a photo of a person holding a notebook with a handwritten page", + "query": "notebook writing talent dreams", + "dia_id": "D17:14", + "text": "I will! I actually started on a book recently since my movie did well!" + }, + { + "speaker": "Nate", + "dia_id": "D17:15", + "text": "Nice! I'm curious, what is it about?" + }, + { + "speaker": "Joanna", + "dia_id": "D17:16", + "text": "That page specifically has some dialogues exploring loss, redemption, and forgiveness. It's a deep and emotional story that I'm really excited about!" + }, + { + "speaker": "Nate", + "dia_id": "D17:17", + "text": "Wow, Joanna! It sounds awesome. I'm so excited to see how it all plays out!" + }, + { + "speaker": "Joanna", + "dia_id": "D17:18", + "text": "Thanks, Nate! I'm so glad you're excited. I've never really tried publishing a book, but this might be the first!" + }, + { + "speaker": "Nate", + "dia_id": "D17:19", + "text": "Good luck on that! I'm sure people will recognise you as the same author of the movie you got published and love the book even more." + }, + { + "speaker": "Joanna", + "dia_id": "D17:20", + "text": "Thanks, Nate! Your belief in me means a lot. I'll keep doing my best. Thanks for the support!" + }, + { + "speaker": "Nate", + "blip_caption": "a photo of a person holding a small turtle in a container", + "dia_id": "D17:21", + "text": "No problem, Joanna. I'm here for you. Your hard work will pay off, I promise. Believe in yourself and your talent - you're incredible!" + } + ], + "session_18_date_time": "6:12 pm on 14 August, 2022", + "session_18": [ + { + "speaker": "Joanna", + "img_url": [ + "https://i.pinimg.com/originals/2f/df/23/2fdf2357bee9649780477b595cb2b003.jpg" + ], + "blip_caption": "a photo of a notebook with a bunch of stickers on it", + "query": "writing journal ideas notes", + "dia_id": "D18:1", + "text": "Hey Nate, long time no talk! I've been busy with writing projects and really going all out with it. It's been the best thing ever - a mix of highs and lows - and my journal's pretty much my rock. Writing's such a huge part of me now." + }, + { + "speaker": "Nate", + "dia_id": "D18:2", + "text": "Hey Joanna! Great to hear it! It's amazing how much a certain activity can become a part of our lives. Keep it up, you're inspiring! Is writing your way to solace and creativity?" + }, + { + "speaker": "Joanna", + "blip_caption": "a photo of a handwritten letter from a man who is holding a piece of paper", + "dia_id": "D18:3", + "text": "Yeah, definitely. Writing has become like an escape and a way to express my feelings. It gives me a chance to put all my thoughts and feelings down and make something good out of it. Words just have a magical way of healing." + }, + { + "speaker": "Nate", + "dia_id": "D18:4", + "text": "That's really cool, I like it! It's incredible how words can turn something sad into something special. I'm glad it worked for you. Anything cool happening recently?" + }, + { + "speaker": "Joanna", + "img_url": [ + "https://i.redd.it/6a9y8ycad0ga1.jpg" + ], + "blip_caption": "a photo of a note written to a person on a piece of paper", + "query": "handwritten letter impact writing", + "dia_id": "D18:5", + "text": "Yep. Last week, someone wrote me a letter after reading an online blog post I made about a hard moment in my life. Their words touched me; they said my story had brought them comfort. It was awesome to realize my words had that kind of power. It reminded me why I love writing." + }, + { + "speaker": "Nate", + "dia_id": "D18:6", + "text": "Nice work, Joanna! That must feel sureal. Keep it up - you're changing lives!" + }, + { + "speaker": "Joanna", + "dia_id": "D18:7", + "text": "Thanks, Nate! Really appreciate your kind words. It's knowing that my writing can make a difference that keeps me going, even on tough days. So glad to have this outlet to share my stories and hopefully have an impact. How about you? Anything new since we last talked?" + }, + { + "speaker": "Nate", + "img_url": [ + "https://i2.pickpik.com/photos/798/109/80/ice-cream-sundae-ice-cream-sundae-vanilla-preview.jpg" + ], + "blip_caption": "a photography of a dessert with whipped cream and chocolate sauce", + "query": "coconut milk ice cream homemade toppings", + "dia_id": "D18:8", + "re-download": true, + "text": "Thanks, Joanna! Your words mean a lot. Since we last spoke, I started teaching people how to make this. Sharing my love for dairy-free desserts has been fun and rewarding." + }, + { + "speaker": "Joanna", + "dia_id": "D18:9", + "text": "Yum, Nate! I love it when you make coconut milk icecream, it's so good!" + }, + { + "speaker": "Nate", + "dia_id": "D18:10", + "text": "I've been really into making this lately - it's creamy, rich, dairy-free and a new recipe! Wanna try it? I can share the recipe with you if you'd like!" + }, + { + "speaker": "Joanna", + "dia_id": "D18:11", + "text": "I'd love to try it! Thanks for sharing your love for dairy-free desserts. I really appreciate it!" + }, + { + "speaker": "Nate", + "dia_id": "D18:12", + "text": "No problem, Joanna! Always happy to share them with you. Sending you the recipe now!" + }, + { + "speaker": "Joanna", + "dia_id": "D18:13", + "text": "Thanks, Nate! Can't wait to surprise my family with something delicious!" + }, + { + "speaker": "Nate", + "dia_id": "D18:14", + "text": "No problem, Joanna! Wish them luck! Let me know how it goes. Have a blast baking!" + }, + { + "speaker": "Joanna", + "dia_id": "D18:15", + "text": "Thanks, Nate. I'll keep you posted. Have a great day." + }, + { + "speaker": "Nate", + "dia_id": "D18:16", + "text": "You too!" + } + ], + "session_19_date_time": "10:57 am on 22 August, 2022", + "session_19": [ + { + "speaker": "Nate", + "dia_id": "D19:1", + "text": "Woah Joanna, I won an international tournament yesterday! It was wild. Gaming has brought me so much success and now I'm able to make a living at something I'm passionate about - I'm loving it." + }, + { + "speaker": "Joanna", + "dia_id": "D19:2", + "text": "Congrats, Nate! So proud of you for winning that tournament, that's awesome! Must feel great to turn your passion into a career." + }, + { + "speaker": "Nate", + "img_url": [ + "https://i.pinimg.com/originals/57/31/42/57314282992ff77a40be8450c003b5c7.jpg" + ], + "blip_caption": "a photo of a fish tank with a fish inside of it", + "query": "cute pet turtles tank", + "dia_id": "D19:3", + "text": "I'm really stoked to see all my hard work paying off! I'm super proud of what I accomplished. On another note, my little dudes got a new tank! Check them out, they're so cute, right?!" + }, + { + "speaker": "Joanna", + "dia_id": "D19:4", + "text": "Wow Nate, they're adorable! I can see why you enjoy spending time with them. It looks like they have so much more room to swim now!" + }, + { + "speaker": "Nate", + "dia_id": "D19:5", + "text": "They're my little buddies, always calm and peaceful. It makes coming home after a long day of gaming better. The tank expansion has made them so happy! How have you been?" + }, + { + "speaker": "Joanna", + "img_url": [ + "https://halekatiedotcom.files.wordpress.com/2020/09/img_6976.jpg" + ], + "blip_caption": "a photo of a desk with a chair and a computer", + "query": "writers group meeting stack scripts desk", + "dia_id": "D19:6", + "text": "I'm good! Was super nervous last week when I shared my book with my writers group but got some great feedback. My hard work is paying off, it's such an awesome feeling!" + }, + { + "speaker": "Nate", + "blip_caption": "a photo of a dessert in a glass on a counter", + "dia_id": "D19:7", + "text": "Wow Jo, you're killing it! Getting this kind of feedback means people are really connecting with your writing. Pretty cool! Did you celebrate?" + }, + { + "speaker": "Joanna", + "img_url": [ + "https://tastecando.com/cdn/shop/articles/Keto_Krisp_January_Post.jpg" + ], + "blip_caption": "a photo of two desserts with spoons and a bar of chocolate", + "query": "raspberry chia pudding parfait dessert", + "dia_id": "D19:8", + "text": "Thanks, Nate! It feels great knowing that people like my writing. I celebrated by making this delicious treat - yum! Any plans for the weekend?" + }, + { + "speaker": "Nate", + "dia_id": "D19:9", + "text": "I'm taking some time off this weekend to chill with my pets. Anything cool happening with you?" + }, + { + "speaker": "Joanna", + "dia_id": "D19:10", + "text": "I'm relaxing and recharging this weekend with a long walk and some reading. It's a good break." + }, + { + "speaker": "Nate", + "dia_id": "D19:11", + "text": "Looks like we both need a break. I'm glad your able to find a way to recharge! It's so incredibly important to take time off!" + }, + { + "speaker": "Joanna", + "dia_id": "D19:12", + "text": "Thanks, Nate! I've learned that taking breaks and looking after myself are important for my inspiration and mental health. It's all about finding balance." + }, + { + "speaker": "Nate", + "img_url": [ + "https://i.redd.it/k78gqk5c5kx71.jpg" + ], + "blip_caption": "a photo of a bookcase filled with books and a toy car", + "query": "bookshelf filled with books", + "dia_id": "D19:13", + "text": "Yeah, balance is key! It's so cool how taking care of ourselves helps us be more creative and happier. I'm always looking for something new to read. Got any book recommendations? I've got a lot of books to choose from." + }, + { + "speaker": "Joanna", + "dia_id": "D19:14", + "text": "I reccomend finding a fantasy book series to read through. Most fiction series are great reads when your trying to relax." + }, + { + "speaker": "Nate", + "img_url": [ + "https://thelitbitch.files.wordpress.com/2023/04/img_2043_jpg.jpg" + ], + "blip_caption": "a photo of a stack of books sitting on top of a wooden table", + "query": "a court of thorns and roses book sarah j maas", + "dia_id": "D19:15", + "text": "Good idea! How about this series?" + }, + { + "speaker": "Joanna", + "dia_id": "D19:16", + "text": "That's a great one! Let me know what you think when your finished!" + }, + { + "speaker": "Nate", + "img_url": [ + "https://i.redd.it/vx7o8gcqv01c1.jpg" + ], + "blip_caption": "a photo of a poster of a man falling off a cliff", + "query": "space opera book series", + "dia_id": "D19:17", + "text": "Sure thing! And since your recommending me a book, I thought I should do the same! I'd really recommend this series. It's got awesome battles and interesting characters." + }, + { + "speaker": "Joanna", + "dia_id": "D19:18", + "text": "Wow, that series looks awesome! I'll have to check it out sometime!" + }, + { + "speaker": "Nate", + "dia_id": "D19:19", + "text": "You really should! The action scenes are awesome and the plot rocks. Definitely one of my favorites!" + }, + { + "speaker": "Joanna", + "dia_id": "D19:20", + "text": "Wow, sounds great! I'll definitely add it to my list; thanks for the recommendation!" + }, + { + "speaker": "Nate", + "dia_id": "D19:21", + "text": "Enjoy it! Have a good day." + }, + { + "speaker": "Joanna", + "dia_id": "D19:22", + "text": "Thanks, Nate! You too! Have a great day. Take care." + } + ], + "session_20_date_time": "6:03 pm on 5 September, 2022", + "session_20": [ + { + "speaker": "Nate", + "img_url": [ + "https://live.staticflickr.com/8056/8444279059_bbf0c79356_b.jpg" + ], + "blip_caption": "a photography of two turtles sitting on a rock in a pond", + "query": "pet turtles mario luigi", + "dia_id": "D20:1", + "re-download": true, + "text": "Hey Joanna! Long time no talk. So much has happened. Look how cute they are! Hanging with them has been a big help, especially recently. Speaking of which, I just had a letdown in a video game tourney - I didn't do too great, even though I tried. It was a setback, but I'm trying to stay positive." + }, + { + "speaker": "Joanna", + "img_url": [ + "https://chensplate.com/wp-content/uploads/2021/02/IMG_8512.jpg" + ], + "blip_caption": "a photo of a piece of cake with strawberries and chocolate", + "query": "dairy-free chocolate cake pink frosting", + "dia_id": "D20:2", + "text": "Hey Nate! Cute turtles! Bummer about the setback. Any positive vibes comin' your way? I just revised on of my old recipes and made this!" + }, + { + "speaker": "Nate", + "dia_id": "D20:3", + "text": "Hey Joanna, yeah it's a bummer that I didn't do well. But it's all part of the learning curve, you know? Also that looks super good! Anyways, how are you holding up?" + }, + { + "speaker": "Joanna", + "dia_id": "D20:4", + "text": "I'm doing OK, thanks. Just been tinkering with that recipe and a few others. It's helping me find some comfort and getting creative." + }, + { + "speaker": "Nate", + "dia_id": "D20:5", + "text": "What else are you making? It's always satisfying to see the kind of things you do when your in one of those moods!" + }, + { + "speaker": "Joanna", + "dia_id": "D20:6", + "text": "Been tweaking a dessert recipe to make it yummier and more accessible." + }, + { + "speaker": "Nate", + "dia_id": "D20:7", + "text": "Wow, that sounds great! What flavors are you experimenting with?" + }, + { + "speaker": "Joanna", + "dia_id": "D20:8", + "text": "Trying out different flavors like chocolate, raspberry, and coconut has been a blast!" + }, + { + "speaker": "Nate", + "dia_id": "D20:9", + "text": "Sounds delicious! Are you only trying dairy-free options?" + }, + { + "speaker": "Joanna", + "img_url": [ + "https://cook2nourish.com/wp-content/uploads/2019/03/Mr6bqaNkSazxNPIxqCHyw-e1553202400166.jpg" + ], + "blip_caption": "a photo of a plate of cupcakes with different toppings", + "query": "dairy-free chocolate coconut cupcakes with raspberry frosting", + "dia_id": "D20:10", + "text": "Yeah, since I'm lactose intolerant I'm trying out dairy-free options like coconut or almond milk instead. It's been a fun challenge seeing how to make yummy treats that suit everyone's diets. I even made these dairy-free chocolate coconut cupcakes with raspberry frosting." + }, + { + "speaker": "Nate", + "dia_id": "D20:11", + "text": "Woah, those look great, Joanna! It's cool that you make desserts that work for everyone's diets. Do you have any more yummy recipes hiding in there?" + }, + { + "speaker": "Joanna", + "dia_id": "D20:12", + "text": "Yep! I've been making all sorts of desserts that work for everyone's diets - cookies, pies, cakes - everything! I'll share more recipes with you soon." + }, + { + "speaker": "Nate", + "blip_caption": "a photo of a pile of cookies with sprinkles on a wooden table", + "dia_id": "D20:13", + "text": "Can't wait to try them. Can I join you sometime? I think baking and cooking really brings us together!" + }, + { + "speaker": "Joanna", + "dia_id": "D20:14", + "text": "Yeah, Nate! A fellow Chef in the kitchen is always a great help! Speaking of which, i'm curious, any more tips for dairy-free baking?" + }, + { + "speaker": "Nate", + "dia_id": "D20:15", + "text": "You can use stuff like dairy-free margarine or coconut oil instead of butter, and make sure to check the labels to ensure they're dairy-free. Good luck!" + }, + { + "speaker": "Joanna", + "blip_caption": "a photo of a cookie with chocolate drizzle and almonds", + "dia_id": "D20:16", + "text": "Thanks, Nate! Love your ideas, can't wait to try them out!" + }, + { + "speaker": "Nate", + "dia_id": "D20:17", + "text": "No problem! I love to help, so just shoot me a question anytime you need!" + }, + { + "speaker": "Joanna", + "dia_id": "D20:18", + "text": "Got it Nate, I'll see you soon!" + }, + { + "speaker": "Nate", + "dia_id": "D20:19", + "text": "Bye Joanna! Good luck!" + } + ], + "session_21_date_time": "1:43 pm on 14 September, 2022", + "session_21": [ + { + "speaker": "Joanna", + "dia_id": "D21:1", + "text": "Hey Nate, long time no see! My laptop crashed last week and I lost all my work - super frustrating! As a writer, my laptop is like half of my lifeline so losing all progress was like a major blow." + }, + { + "speaker": "Nate", + "dia_id": "D21:2", + "text": "Hey Joanna, sorry to hear about that. Losing so much progress must be really frustrating. Did you manage to recover anything? Maybe consider backing up your work in the future?" + }, + { + "speaker": "Joanna", + "dia_id": "D21:3", + "text": "Thanks for the sympathy, Nate. Nothing was recoverable, but now I have an external drive for backups. I never want to go through this again. So, how have you been? Making anything cool?" + }, + { + "speaker": "Nate", + "dia_id": "D21:4", + "text": "Hey Joanna, I'm no writer like you, but something pretty awesome happened. Last Monday I got to teach people vegan ice cream recipes on my own cooking show! It was a bit nerve-wracking to put myself out there, but it was a blast. Plus, I picked up a few new recipes!" + }, + { + "speaker": "Joanna", + "dia_id": "D21:5", + "text": "Way to go, Nate! Congrats on the cooking show, I'll definitely be tuning in! What's your favorite dish from the show?" + }, + { + "speaker": "Nate", + "img_url": [ + "https://live.staticflickr.com/7055/6945130989_40895e5372_b.jpg" + ], + "blip_caption": "a photography of a bowl of ice cream with a spoon on a table", + "query": "coconut milk ice cream", + "dia_id": "D21:6", + "re-download": true, + "text": "Coconut milk ice cream is at the top of my list. It's so smooth and creamy with a tropical coconut twist. Plus, it's dairy-free for people who can't have lactose or who want vegan options. Here's a snap of the ice cream I made." + }, + { + "speaker": "Joanna", + "dia_id": "D21:7", + "text": "Wow, that looks amazing, Nate! I love the color and texture. It's great that you're making these options. Could you share the recipe? I'd love to try making it sometime!" + }, + { + "speaker": "Nate", + "dia_id": "D21:8", + "text": "Yeah sure! Would love to share it. Let's spread the joy of dairy-free options! Let me know when you make it!" + }, + { + "speaker": "Joanna", + "dia_id": "D21:9", + "text": "Cool, Nate! Gonna give it a go. Dairy-free is a must for me, especially for desserts. Last Friday, I made a deeeelish dessert with almond milk - it was good! Got any favs when it comes to dairy-free desserts?" + }, + { + "speaker": "Nate", + "dia_id": "D21:10", + "text": "Coconut milk ice cream is one of my favorites as you might be able to tell, but I also love a dairy-free chocolate mousse. It's super creamy and tastes like the real thing. What's been your favorite dairy-free sweet treat so far?" + }, + { + "speaker": "Joanna", + "img_url": [ + "https://wornslapout.com/wp-content/uploads/2018/11/img_3684.jpg" + ], + "blip_caption": "a photo of a chocolate tart with raspberries on top", + "query": "chocolate raspberry tart almond flour crust ganache fresh raspberries", + "dia_id": "D21:11", + "text": "Hey Nate, my favorite dairy-free treat is this amazing chocolate raspberry tart. It has an almond flour crust, chocolate ganache, and fresh raspberries - it's delicious!" + }, + { + "speaker": "Nate", + "blip_caption": "a photo of a piece of chocolate cake with raspberries on a plate", + "dia_id": "D21:12", + "text": "That looks amazing, Joanna! I need to try baking that. What other treats do you like making?" + }, + { + "speaker": "Joanna", + "img_url": [ + "https://yaycakeday.files.wordpress.com/2019/08/ice-cream-cake-1.jpg" + ], + "blip_caption": "a photo of a piece of chocolate cake with raspberries on a plate", + "query": "dairy-free chocolate cake raspberries", + "dia_id": "D21:13", + "text": "Hey Nate, I love making this dairy-free chocolate cake with raspberries. It's so moist and delicious - perfect sweetness level." + }, + { + "speaker": "Nate", + "dia_id": "D21:14", + "text": "That cake looks amazing, Joanna! How did you make it?" + }, + { + "speaker": "Joanna", + "dia_id": "D21:15", + "text": "I make it with almond flour, coconut oil, chocolate and raspberries. It's my favorite for birthdays and special days." + }, + { + "speaker": "Nate", + "dia_id": "D21:16", + "text": "Yum, Joanna! Gotta try that one. Any others you want to share?" + }, + { + "speaker": "Joanna", + "img_url": [ + "https://tamboracai.com/assets/Megan-Marlow-Acai-Vegan-Cheesecake-Bars_02.jpg" + ], + "blip_caption": "a photo of a piece of cake with walnuts on top", + "query": "dairy-free blueberry cheesecake bars recipe", + "dia_id": "D21:17", + "text": "Hey Nate! Here's another recipe I like. It's a delicious dessert made with blueberries, coconut milk, and a gluten-free crust. So creamy and delicious!" + }, + { + "speaker": "Nate", + "dia_id": "D21:18", + "text": "Wow, Joanna! That dessert looks amazing. I'll definitely have to give it a try. Thanks!" + }, + { + "speaker": "Joanna", + "dia_id": "D21:19", + "text": "Glad to help, Nate. Let me know if you try it. I'm sure you'll enjoy it. It was great chatting." + }, + { + "speaker": "Nate", + "dia_id": "D21:20", + "text": "Thanks Joanna! I will. Bye!" + } + ], + "session_22_date_time": "11:15 am on 6 October, 2022", + "session_22": [ + { + "speaker": "Joanna", + "img_url": [ + "https://i0.wp.com/mittsandmeasures.com/wp-content/uploads/2019/10/img_0051.jpg" + ], + "blip_caption": "a photo of a tart with raspberries on a white plate", + "query": "dairy-free chocolate raspberry tarts", + "dia_id": "D22:1", + "text": "Hey Nate, hi! Yesterday, I tried my newest dairy-free recipe and it was a winner with my family! Mixing and matching flavors is fun and I'm always trying new things. How about you?" + }, + { + "speaker": "Nate", + "img_url": [ + "https://i.redd.it/0z7nrwjeqc431.jpg" + ], + "blip_caption": "a photo of a trophy and a game controller on a table", + "query": "video game tournament trophy cash prize", + "dia_id": "D22:2", + "text": "Hey Joanna! That tart looks yummy! Lately, I've been doing great - I won a really big video game tournament last week and it was awesome! I still can't believe I made so much money from it." + }, + { + "speaker": "Joanna", + "dia_id": "D22:3", + "text": "Way to go, Nate! Winning the tournament and earning cash is awesome - congrats! Did you save it for something special?" + }, + { + "speaker": "Nate", + "dia_id": "D22:4", + "text": "Thanks Joanna! Yeah, I saved some but I'm not sure what to do with it - I'm completely content already. I don't have big plans anyway, so it's nice to have the extra cash on hand." + }, + { + "speaker": "Joanna", + "dia_id": "D22:5", + "text": "That's awesome, Nate! Having some extra cash on hand definitely brings a sense of freedom and relaxation, huh?" + }, + { + "speaker": "Nate", + "dia_id": "D22:6", + "text": "Yes! Finally, I don't have to stress about it, so I can just enjoy my movies and games." + }, + { + "speaker": "Joanna", + "dia_id": "D22:7", + "text": "Taking breaks and reducing stress is pretty nice! Have you watched any good movies recently? I could use some recommendations!" + }, + { + "speaker": "Nate", + "dia_id": "D22:8", + "text": "I watched \"Little Women\" recently, and it was great! The acting was awesome and the story was so captivating. Definitely a good one!" + }, + { + "speaker": "Joanna", + "dia_id": "D22:9", + "text": "I'm so glad you enjoyed it! I recommended it to you a while back. I watched it too and it really spoke to me. Themes like sisterhood, love, and chasing dreams were explored so well. By the way, I finished up my writing for my book last week. Put in a ton of late nights and edits but finally got it done. I'm so proud of it! Can't wait to see what happens next." + }, + { + "speaker": "Nate", + "dia_id": "D22:10", + "text": "Way to go! We both know it took some effort, but I'm sure it'll be great. Congrats on finishing it up!" + }, + { + "speaker": "Joanna", + "dia_id": "D22:11", + "text": "Thanks Nate! Your words mean a lot. Dedication and late nights got me here, but it was worth it. Just like you with your recent tournament - hard work pays off. I appreciate your support throughout!" + }, + { + "speaker": "Nate", + "dia_id": "D22:12", + "text": "I'm always here for you, Joanna! You've worked so hard and accomplished a lot – I'm proud. Keep on going!" + }, + { + "speaker": "Joanna", + "dia_id": "D22:13", + "text": "Thanks, Nate! I won't give up on my goals as long as your here to support me." + }, + { + "speaker": "Nate", + "img_url": [ + "https://i.pinimg.com/originals/5e/f4/f9/5ef4f9bd2f094b2d0a4ddd4861b928a0.jpg" + ], + "blip_caption": "a photo of a white board with a drawing of arrows and words", + "query": "motivational quotes whiteboard", + "dia_id": "D22:14", + "text": "You can always count on me! I even made this for you!" + }, + { + "speaker": "Joanna", + "dia_id": "D22:15", + "text": "Wow, Nate, that looks awesome! What inspired you?" + }, + { + "speaker": "Nate", + "blip_caption": "a photo of a young boy drawing on a white board", + "dia_id": "D22:16", + "text": "I figured you could always look back on this whenever you need encouragement, and that was all the inspiration I needed. And I would also say that your life path can be quite inspirational!" + }, + { + "speaker": "Joanna", + "dia_id": "D22:17", + "text": "Wow, Nate! That's sweet of you! I'll make sure to remember this when I need the encouragement the most." + }, + { + "speaker": "Nate", + "dia_id": "D22:18", + "text": "Awesome! I know encouragement is what got me so far in my gameing career, so I figured why not share the love." + }, + { + "speaker": "Joanna", + "img_url": [ + "https://i.pinimg.com/originals/7e/d3/bb/7ed3bb9129ef6126259592e038e2cd17.jpg" + ], + "blip_caption": "a photo of a bookmark with a plant on top of it", + "query": "stack of books bookmark", + "dia_id": "D22:19", + "text": "Rest assured, it will be something I cherish! On another note, I just finished this cute little bookmark for one of the ladies at my writing club!" + }, + { + "speaker": "Nate", + "dia_id": "D22:20", + "text": "That bookmark is great. I'm sure she'll love it!" + }, + { + "speaker": "Joanna", + "dia_id": "D22:21", + "text": "Thanks Nate! I absolutley love DIYs, and I know she does too." + }, + { + "speaker": "Nate", + "dia_id": "D22:22", + "text": "Let me know how it goes!" + }, + { + "speaker": "Joanna", + "dia_id": "D22:23", + "text": "Sure thing! Bye for now!" + } + ], + "session_23_date_time": "10:58 am on 9 October, 2022", + "session_23": [ + { + "speaker": "Nate", + "img_url": [ + "http://scarlet-rhapsody.com/wp-content/uploads/2020/03/87613523_2854559627994666_1775718911015124992_n.jpg" + ], + "blip_caption": "a photo of a man standing in front of a table with board games", + "query": "gaming convention group gamers", + "dia_id": "D23:1", + "text": "Hey Joanna, it's been a couple days since we last talked. Something exciting happened last Friday. I went to a game convention and met new people who weren't from my normal circle. It was a bit overwhelming but it reminded me of the good times gaming can bring." + }, + { + "speaker": "Joanna", + "dia_id": "D23:2", + "text": "Hey Nate! Good to hear from you. Sounds like fun! Meeting new people can be overwhelming, but the rewards can be great. Sometimes it's good to step outside our comfort zones and explore new things." + }, + { + "speaker": "Nate", + "img_url": [ + "https://i.redd.it/omjzdx4vo4h41.jpg" + ], + "blip_caption": "a photo of two people dressed up in costumes posing for a picture", + "query": "video game convention friends", + "dia_id": "D23:3", + "text": "Yeah, you're right! We can have great experiences if we take risks. I even made some friends at the convention who love games just like me. We already planned a gaming session together - it's cool to meet people who have the same interests!" + }, + { + "speaker": "Joanna", + "dia_id": "D23:4", + "text": "That looks awesome! I'm glad you met people who share your interests - that definitely makes experiences more fun.." + }, + { + "speaker": "Nate", + "img_url": [ + "https://i.redd.it/n5hled3cve841.jpg" + ], + "blip_caption": "a photo of a group of people sitting around a table playing a board game", + "query": "group of people playing game", + "dia_id": "D23:5", + "text": "I also met some people who also played this boardgame I love, so I joined in. We had a lot in common and hit it off. It's great when shared hobbies can bond people!" + }, + { + "speaker": "Joanna", + "dia_id": "D23:6", + "text": "It's incredible how a game can bring people together and form strong relationships. Did you do anything else there?" + }, + { + "speaker": "Nate", + "img_url": [ + "https://i.redd.it/byq7wy33k4w11.jpg" + ], + "blip_caption": "a photo of a board game with a blue board and a yellow board", + "query": "catan board game pieces strategy", + "dia_id": "D23:7", + "text": "We played this game Catan - it's a great strategy game where you build settlements and trade resources. I love it!" + }, + { + "speaker": "Joanna", + "dia_id": "D23:8", + "text": "Looks cool! Is it more of a competitive game or a more chill one?" + }, + { + "speaker": "Nate", + "dia_id": "D23:9", + "text": "It can be both competitive and chill. We were competing, but still had lots of fun." + }, + { + "speaker": "Joanna", + "dia_id": "D23:10", + "text": "Competitive games can definitely be difficult to play sometimes when people get all upset about every move you make, but it's cool they were all chill! Glad you had a good time." + }, + { + "speaker": "Nate", + "dia_id": "D23:11", + "text": "It was great! Playing games is my escape from life struggles, so I generally don't get crazy competitive over them. The people at the convention were the same way!" + }, + { + "speaker": "Joanna", + "dia_id": "D23:12", + "text": "Glad you found a way to have fun and escape! It's important to stay happy and de-stress. Keep doing what makes you happy!" + }, + { + "speaker": "Nate", + "dia_id": "D23:13", + "text": "For sure! You should go to a writing convention or something sometime! The experience at this convention is unforgetable." + }, + { + "speaker": "Joanna", + "dia_id": "D23:14", + "text": "Do writing conventions exist? I'll have to look into that, it could be fun! Thanks for the idea. Have you been up to anything tonight?" + }, + { + "speaker": "Nate", + "img_url": [ + "https://i.redd.it/806azldnt6bb1.jpg" + ], + "blip_caption": "a photo of a bookcase filled with dvds and movies", + "query": "video games movie collection", + "dia_id": "D23:15", + "text": "Mostly just chilling at home. Playing video games or watching movies helps me unwind." + }, + { + "speaker": "Joanna", + "dia_id": "D23:16", + "text": "That sounds great! What's your favorite game or movie that you've seen recently?" + }, + { + "speaker": "Nate", + "dia_id": "D23:17", + "text": "I recently saw a movie that blew my mind with all the twists and dream stuff, I think it was called \"Inception\". I've also been playing a game nonstop with a great futuristic setting and gameplay called \"Cyberpunk 2077\". Have you seen or played anything good lately?" + }, + { + "speaker": "Joanna", + "img_url": [ + "http://mjbmemorabilia.com/cdn/shop/files/1_4c670c88-d207-4080-9b81-9b81ef1066fe.jpg" + ], + "blip_caption": "a photo of a framed movie poster with a signed picture", + "query": "classic movie captivating storyline amazing performances movie poster", + "dia_id": "D23:18", + "text": "I watched a classic movie the other day that was awesome - the story was so gripping and the actors were great! It really stuck with me." + }, + { + "speaker": "Nate", + "dia_id": "D23:19", + "text": "Wow, that must have been awesome! What would you rate it?" + }, + { + "speaker": "Joanna", + "dia_id": "D23:20", + "text": "Well, it was amazing, so probably 9 or 10 out of 10! Movies can take us to different places and make us feel lots of emotions. What do you love about watching them?" + }, + { + "speaker": "Nate", + "dia_id": "D23:21", + "text": "Well they take me to new worlds and fill me with emotions. Plus, they're great for chilling out after a day." + }, + { + "speaker": "Joanna", + "img_url": [ + "https://i.pinimg.com/originals/1a/02/2c/1a022c736bdfb7c47f33272c27eac0e7.jpg" + ], + "blip_caption": "a photo of a living room with a tv and candles", + "query": "cozy movie night setup popcorn blankets", + "dia_id": "D23:22", + "text": "I agree! They have the power to take us away and make us feel things not normally experienced in life. It's a great escape! Especially when you have a room like this!" + }, + { + "speaker": "Nate", + "dia_id": "D23:23", + "text": "Wow, it's so comfy in there! I should really get a set up like that." + }, + { + "speaker": "Joanna", + "dia_id": "D23:24", + "text": "You should! It's one thing to watch a movie in a theater, but on a nice comfy couch with a good blanket, I feel so at peace!" + }, + { + "speaker": "Nate", + "dia_id": "D23:25", + "text": "Any pointers on what I should get for my living room to make it comfy like that?" + }, + { + "speaker": "Joanna", + "blip_caption": "a photo of a poster of a man and a woman sitting on a bench", + "dia_id": "D23:26", + "text": "Sure! For one, you should get a couch that can sit multiple people so that you can lay down if you want, and make sure its really fluffy! Also invest in a blanket that has a little bit of weight to it, and some lights that can be dimmed." + }, + { + "speaker": "Nate", + "dia_id": "D23:27", + "text": "Sounds like you really got into making your living room! Thanks, I'll try that out for myself!" + }, + { + "speaker": "Joanna", + "dia_id": "D23:28", + "text": "No problem! I get super invested in random little things like that, but I think its all worth it when I end up with random little things that make life so much nicer." + }, + { + "speaker": "Nate", + "dia_id": "D23:29", + "text": "Thanks for the tip! See you later!" + }, + { + "speaker": "Joanna", + "dia_id": "D23:30", + "text": "See ya Nate!" + } + ], + "session_24_date_time": "2:01 pm on 21 October, 2022", + "session_24": [ + { + "speaker": "Nate", + "dia_id": "D24:1", + "text": "Hey Joanna, what's been up? Haven't seen you since we last talked." + }, + { + "speaker": "Joanna", + "dia_id": "D24:2", + "text": "Hey Nate! I have been revising and perfecting the recipe I made for my family and it turned out really tasty. What's been happening with you?" + }, + { + "speaker": "Nate", + "dia_id": "D24:3", + "text": "Hey Joanna! That's cool. I've been getting so stressed lately because of my tournament progress - tough competitors - but my turtles always cheer me up." + }, + { + "speaker": "Joanna", + "blip_caption": "a photo of a cat laying on the floor in a room", + "dia_id": "D24:4", + "text": "Pets have a way of brightening our days. I still have that stuffed animal dog you gave me! I named her Tilly, and she's always with me while I write." + }, + { + "speaker": "Nate", + "dia_id": "D24:5", + "text": "Glad to hear it! What made you name her Tilly?" + }, + { + "speaker": "Joanna", + "dia_id": "D24:6", + "text": "I used to have a dog back in Michigan with that name, but then I got allergic and we had to get rid of her. The name helps me remember her back when I used to be able to hold and squeeze animal without an allergic reaction!" + }, + { + "speaker": "Nate", + "dia_id": "D24:7", + "text": "That's so touching! Glad the stuffed animal means so much!" + }, + { + "speaker": "Joanna", + "dia_id": "D24:8", + "text": "Tilly helps me stay focused and brings me so much joy. It's amazing how even stuffed animals can do that!" + }, + { + "speaker": "Nate", + "dia_id": "D24:9", + "text": "It really is, I'm not sure I'll ever understand why watching my turtles slowly walk around makes me so happy. But I'm very glad it does." + }, + { + "speaker": "Joanna", + "dia_id": "D24:10", + "text": "Same here! So have you been up to anything recenly?" + }, + { + "speaker": "Nate", + "dia_id": "D24:11", + "text": "Yeah, I've just been practicing for my next video game tournemant. How about you?" + }, + { + "speaker": "Joanna", + "dia_id": "D24:12", + "text": "Well, I had a bit of a setback recently - another rejection from a production company." + }, + { + "speaker": "Nate", + "dia_id": "D24:13", + "text": "Bummer, Joanna. Is this the one you sent to a film contest? Rejections suck, but don't forget they don't define you. Keep at it and you'll find the perfect opportunity." + }, + { + "speaker": "Joanna", + "dia_id": "D24:14", + "text": "Yeah.. Thanks, Nate. It's hard, but I won't let it slow me down. I'm gonna keep grinding and moving ahead." + }, + { + "speaker": "Nate", + "dia_id": "D24:15", + "text": "That's what I like to hear! I really respect you for that and being able to bounce back whenever something sad happens!" + }, + { + "speaker": "Joanna", + "dia_id": "D24:16", + "text": "Thanks, Nate! Your encouragement really means a lot. I'm gonna keep pushing forward and believing in myself." + }, + { + "speaker": "Nate", + "dia_id": "D24:17", + "text": "You got this, and don't ever forget that you have people cheering you on from the sidelines wherever you go." + }, + { + "speaker": "Joanna", + "dia_id": "D24:18", + "text": "Thanks! I'll see you around!" + }, + { + "speaker": "Nate", + "dia_id": "D24:19", + "text": "No problem! Catch you later!" + } + ], + "session_25_date_time": "8:16 pm on 25 October, 2022", + "session_25": [ + { + "speaker": "Nate", + "dia_id": "D25:1", + "text": "Hey Joanna, what's been up since we last chatted? How's it going?" + }, + { + "speaker": "Joanna", + "blip_caption": "a photo of a box of cards with a quote on it", + "dia_id": "D25:2", + "text": "Hey Nate! Another movie script that I contributed to was shown on the big screen last Sunday for the first time! It was such a surreal experience to see everything come together. I felt a mix of emotions, but overall, it was a satisfying moment. I've been waiting for this for a long time!" + }, + { + "speaker": "Nate", + "blip_caption": "a photo of a box with a controller inside of it", + "dia_id": "D25:3", + "text": "Congrats Joanna! How was it to finally see it on the big screen?\n\n[shares a photo holding a videogame controller]" + }, + { + "speaker": "Joanna", + "dia_id": "D25:4", + "text": "It was an amazing experience! I'll never forget seeing all of the characters and dialogue I wrote being acted out - it was such a cool feeling. Having all the hard work and determination I put into writing pay off was definitely rewarding. I know this is the third time it's happened, but its just so awesome!" + }, + { + "speaker": "Nate", + "dia_id": "D25:5", + "text": "That must have been amazing. What was your favorite part of it?" + }, + { + "speaker": "Joanna", + "img_url": [ + "https://i.redd.it/ioa9eysnr9xb1.jpg" + ], + "blip_caption": "a photo of a drawing book with a bunch of drawings on it", + "query": "character sketches actors notebook", + "dia_id": "D25:6", + "text": "Seeing my characters I worked so hard on come alive was my favorite part. It felt like they jumped off the page and became real - it was totally surreal." + }, + { + "speaker": "Nate", + "dia_id": "D25:7", + "text": "Wow Joanna, those drawings are really incredible! What inspired you to create them?" + }, + { + "speaker": "Joanna", + "dia_id": "D25:8", + "text": "Thanks, Nate! They're visuals of the characters to help bring them alive in my head so I can write better." + }, + { + "speaker": "Nate", + "dia_id": "D25:9", + "text": "That's a cool way to gain insight into your characters. Where did you get your ideas for them?" + }, + { + "speaker": "Joanna", + "dia_id": "D25:10", + "text": "I got ideas from everywhere: people I know, stuff I saw, even what I imagined. It's cool to see how an idea takes shape into a person with their own wants, worries, and wishes." + }, + { + "speaker": "Nate", + "dia_id": "D25:11", + "text": "Wow Joanna, that's so cool! It's amazing how our imaginations can bring ideas to life. Can you tell me more about the character on the left in the photo?" + }, + { + "speaker": "Joanna", + "dia_id": "D25:12", + "text": "Nope! You'll just have to watch the movie and find out for yourself!" + }, + { + "speaker": "Nate", + "dia_id": "D25:13", + "text": "You got it. I was already planning on watching it, but talking to you about it makes me want to watch it even more!" + }, + { + "speaker": "Joanna", + "dia_id": "D25:14", + "text": "Awesome! Well enough about me, what have you been up to?" + }, + { + "speaker": "Nate", + "img_url": [ + "https://cdn12.picryl.com/photo/2016/12/31/water-turtle-on-the-water-animal-animals-a79381-1024.jpg" + ], + "blip_caption": "a photography of two turtles sitting on a log in a pond", + "query": "pet turtles", + "dia_id": "D25:15", + "re-download": true, + "text": "I was bored today, so I just took my turtles out for a walk." + }, + { + "speaker": "Joanna", + "dia_id": "D25:16", + "text": "Sound fun! Did they have a good time?" + }, + { + "speaker": "Nate", + "img_url": [ + "https://upload.wikimedia.org/wikipedia/commons/d/dd/Tortoises_in_the_Cotswold_Wildlife_Park_restaurant_-_geograph.org.uk_-_1468751.jpg" + ], + "blip_caption": "a photography of a dog laying on a rock in a zoo", + "query": "turtles basking heat lamp", + "dia_id": "D25:17", + "re-download": true, + "text": "Of course! They look tired from all the walking, so they're relaxing in the tank right now." + }, + { + "speaker": "Joanna", + "blip_caption": "a photo of a spoon full of ice cream and chocolate sauce", + "dia_id": "D25:18", + "text": "Aww, they're so cute! What do they eat?" + }, + { + "speaker": "Nate", + "blip_caption": "a photo of a container of lettuce and other greens", + "dia_id": "D25:19", + "text": "They eat a combination of vegetables, fruits, and insects. They have a varied diet." + }, + { + "speaker": "Joanna", + "dia_id": "D25:20", + "text": "Wow, that's fascinating! It's interesting how they have such a varied diet, including insects. Do you have a favorite among their food choices?" + }, + { + "speaker": "Nate", + "img_url": [ + "https://images.pexels.com/photos/4460504/pexels-photo-4460504.jpeg" + ], + "blip_caption": "a photography of a group of strawberries and a turtle on a table", + "query": "turtles eating strawberries", + "dia_id": "D25:21", + "re-download": true, + "text": "I love seeing them eat fruit - they get so hyped and it's so cute!" + }, + { + "speaker": "Joanna", + "dia_id": "D25:22", + "text": "Wow, that's so cute! They look like they really enjoy fruit." + }, + { + "speaker": "Nate", + "img_url": [ + "https://i.redd.it/i1d1xxfxqd251.jpg" + ], + "blip_caption": "a photo of a person holding a small turtle in their hand", + "query": "turtles joy holding", + "dia_id": "D25:23", + "text": "Yeah, it's adorable! Watching them enjoy their favorite snacks is so fun. I also like holding them." + }, + { + "speaker": "Joanna", + "blip_caption": "a photo of a rock formation with a person standing on the side", + "dia_id": "D25:24", + "text": "Do they have different personalities, like the way dogs and cats do?" + }, + { + "speaker": "Nate", + "dia_id": "D25:25", + "text": "Yeah, they each do. One is more adventurous while the other is more reserved, which I find cute. Having them around brings me joy and they make great companions." + }, + { + "speaker": "Joanna", + "dia_id": "D25:26", + "text": "That's super cool! I never knew turtles could be so interesting until I met your turtles. Wow!" + }, + { + "speaker": "Nate", + "dia_id": "D25:27", + "text": "I've always liked turtles since I was a boy, so I know all about them!" + }, + { + "speaker": "Joanna", + "dia_id": "D25:28", + "text": "You'll have to keep me posted on them! It's been fun chatting!" + }, + { + "speaker": "Nate", + "dia_id": "D25:29", + "text": "See you later Joanna!" + } + ], + "session_26_date_time": "3:56 pm on 4 November, 2022", + "session_26": [ + { + "speaker": "Joanna", + "dia_id": "D26:1", + "text": "Wow, Nate, I'm on fire! I just set up meetings with movie producers — my dreams are comin' true!" + }, + { + "speaker": "Nate", + "dia_id": "D26:2", + "text": "Wow Joanna, nice work! How did it go with those producer meetings?" + }, + { + "speaker": "Joanna", + "dia_id": "D26:3", + "text": "Thanks, Nate! The meetings went really well. I felt confident discussing my script and vision and they seemed interested and excited. They loved the elements of self-discovery in it. It was so validating to be taken seriously. I'm feeling hopeful and inspired about the future!" + }, + { + "speaker": "Nate", + "blip_caption": "a photo of a television screen showing a game being played", + "dia_id": "D26:4", + "text": "Way to go, Joanna! Putting yourself out there is really brave and winning recognition for your hard work feels great. It's just like when I win a video game tournament - it feels awesome! I'm so proud of you and so glad you're feeling hopeful and inspired." + }, + { + "speaker": "Joanna", + "img_url": [ + "https://fromthepencup.files.wordpress.com/2023/01/img_3860.jpg" + ], + "blip_caption": "a photo of a notebook with a list of things to write", + "query": "notebook filled handwritten pages early writings", + "dia_id": "D26:5", + "text": "Thanks Nate! Your support and encouragement mean a lot. Writing isn't always easy but moments like these make me appreciate it. I'm so thankful for all the opportunities. Last week, I found these old notebooks with my early writings - it was cool to see how far I've come." + }, + { + "speaker": "Nate", + "blip_caption": "a photo of a turtle laying on a bed of rocks and gravel", + "dia_id": "D26:6", + "text": "That's cool! You must love seeing how you've grown as an artist. Is there a favorite piece from your early writings that stands out to you?" + }, + { + "speaker": "Joanna", + "dia_id": "D26:7", + "text": "Yup, I still remember this story from when I was 10. It was about a brave little turtle who was scared but explored the world anyway. Maybe even back then, I was inspired by stories about finding courage and taking risks. It's still a part of my writing today." + }, + { + "speaker": "Nate", + "blip_caption": "a photo of a turtle laying on a bed of rocks and gravel", + "dia_id": "D26:8", + "text": "You obviously have a passion for writing, and it's funny the story was about a turtle! Their resilience is so inspiring! Take courage and keep pushing yourself with your writing. Great job!" + }, + { + "speaker": "Joanna", + "dia_id": "D26:9", + "text": "Thanks, Nate! They make me think of strength and perseverance. They help motivate me in tough times - glad you find that inspiring!" + }, + { + "speaker": "Nate", + "dia_id": "D26:10", + "text": "What can I say, I love turtles. So, what's been happening with you?" + }, + { + "speaker": "Joanna", + "dia_id": "D26:11", + "text": "Hey Nate! Apart from meetings, I'm working on a project - challenging but fulfilling. How about you? What's been going on?" + }, + { + "speaker": "Nate", + "dia_id": "D26:12", + "text": "Just been helping some friends reset their high scores at the international tournament. It's been fun!" + }, + { + "speaker": "Joanna", + "dia_id": "D26:13", + "text": "Wow, sounds like so much fun! You're really passionate about gaming. Have an awesome time and keep helping others with those high scores!" + }, + { + "speaker": "Nate", + "dia_id": "D26:14", + "text": "Thanks! It feels good to use my skills to make a difference." + }, + { + "speaker": "Joanna", + "dia_id": "D26:15", + "text": "I couldn't agree more! Which is why my meetings are so exciting!" + }, + { + "speaker": "Nate", + "img_url": [ + "https://i.redd.it/4q9s9o2607ib1.jpg" + ], + "blip_caption": "a photo of a bowl of ice cream with a spoon in it", + "query": "homemade coconut milk ice cream", + "dia_id": "D26:16", + "text": "On another note, want to come over and try some of this? It's super yummy, just made it yesterday!" + }, + { + "speaker": "Joanna", + "dia_id": "D26:17", + "text": "Mmm, that looks delicious! Is it lactose-free by any chance?" + }, + { + "speaker": "Nate", + "dia_id": "D26:18", + "text": "Yep, I made it with coconut milk so it's lactose-free!" + }, + { + "speaker": "Joanna", + "blip_caption": "a photo of a bowl of ice cream with a spoon in it", + "dia_id": "D26:19", + "text": "Thanks so much, Nate! Sure! I'll come over tomorrow if that's fine." + }, + { + "speaker": "Nate", + "dia_id": "D26:20", + "text": "I don't see why not! I'm not doing anything then, so your completely welcome to!" + }, + { + "speaker": "Joanna", + "dia_id": "D26:21", + "text": "Awesome! I'll bring some of my recipes so we can both share deserts!" + }, + { + "speaker": "Nate", + "dia_id": "D26:22", + "text": "I'd love that! I've been wanting to try some of your chocolate and rasberry cake for a while now." + }, + { + "speaker": "Joanna", + "dia_id": "D26:23", + "text": "You got it! See you tomorrow!" + }, + { + "speaker": "Nate", + "dia_id": "D26:24", + "text": "See you then! Take care!" + } + ], + "session_27_date_time": "8:10 pm on 7 November, 2022", + "session_27": [ + { + "speaker": "Nate", + "dia_id": "D27:1", + "text": "Hey Joanna! Hope you’re doing alright. Crazy thing happened - I was in the final of a big Valorant tournament last Saturday, and I won! It was the best feeling to see my name as the champion. Tournaments really bring out strong emotions in me." + }, + { + "speaker": "Joanna", + "img_url": [ + "https://i.redd.it/om8u4dx20ir71.jpg" + ], + "blip_caption": "a photo of a notebook with a handwritten letter on it", + "query": "notebook handwritten script movie script presentation movie producers", + "dia_id": "D27:2", + "text": "Hey Nate! Congrats on winning the tournament - that's awesome! I know you must have been buzzing! Anyway, I've been working on something exciting too. Last Friday, I finished the presentation for producers - it was tough but it's looking good. What have you been up to?" + }, + { + "speaker": "Nate", + "dia_id": "D27:3", + "text": "Thanks, Joanna! I've been having a blast and preparing for other tournaments, so I've been real busy - but I'm loving it!" + }, + { + "speaker": "Joanna", + "blip_caption": "a photo of a book with a black border and a white title", + "dia_id": "D27:4", + "text": "Sounds like you're really loving life right now! It's great when you find something that suits you. Being busy can be tiring but it's so rewarding in the end. Keep it up!" + }, + { + "speaker": "Nate", + "dia_id": "D27:5", + "text": "Thanks, Joanna! I'm really grateful to have a job I enjoy every day. So anyways, anything new going on in your life?" + }, + { + "speaker": "Joanna", + "dia_id": "D27:6", + "text": "I am writing another movie script! It's a love story with lots of challenges. I've put lots of hard work into it and I'm hoping to get it on the big screen." + }, + { + "speaker": "Nate", + "blip_caption": "a photo of a turtle laying on a bed of rocks and gravel", + "dia_id": "D27:7", + "text": "Woah Joanna, that's incredible! I remember when you started working on these sorta things. It's crazy to see how far you've gotten! You've really got a thing for writing, huh? Where'd you get the idea for it?" + }, + { + "speaker": "Joanna", + "dia_id": "D27:8", + "text": "Thanks Nate! Writing has always been a passion of mine. I got the idea for this script from a dream. How have your turtles been? I haven't seen pictures of them in a while!" + }, + { + "speaker": "Nate", + "dia_id": "D27:9", + "text": "Great actually! These little guys sure bring joy to my life! Watching them is so calming and fascinating. I've really grown fond of them. So, what about you, Joanna? What brings you happiness?" + }, + { + "speaker": "Joanna", + "dia_id": "D27:10", + "text": "Creating stories and watching them come alive gives me happiness and fulfillment. Writing has been such a blessing for me." + }, + { + "speaker": "Nate", + "dia_id": "D27:11", + "text": "Well with dedication like yours, its no wonder you do so well in it as well! Are you planning on submitting anymore scripts anytime soon?" + }, + { + "speaker": "Joanna", + "dia_id": "D27:12", + "text": "Yep! I actually just submitted a few more last week! Hoping to hear back from them soon, though I assume a few will be rejected." + }, + { + "speaker": "Nate", + "dia_id": "D27:13", + "text": "Even if it happens to a few, I'm sure at leasts one will make it to the screens and be your 3rd published movie!" + }, + { + "speaker": "Joanna", + "dia_id": "D27:14", + "text": "Thanks, Nate! Appreciate the encouragement. I won't give up, I promise! Got it covered!" + }, + { + "speaker": "Nate", + "img_url": [ + "https://i.pinimg.com/originals/92/1a/c3/921ac332715f9ecb9f50155427aa8ffc.jpg" + ], + "blip_caption": "a photo of a desk with a computer monitor and a keyboard", + "query": "gaming setup", + "dia_id": "D27:15", + "text": "Great to hear! On another note, I just upgraded some of my equipment at home. Check it out!" + }, + { + "speaker": "Joanna", + "dia_id": "D27:16", + "text": "Oh wow, ice set-up! Do you use that computer for gaming?" + }, + { + "speaker": "Nate", + "dia_id": "D27:17", + "text": "Yep! This is where I practice and compete. Sometimes I even use it when I'm playing games with friends." + }, + { + "speaker": "Joanna", + "dia_id": "D27:18", + "text": "Cool! Having a dedicated space for practice and competition should help you stay focused." + }, + { + "speaker": "Nate", + "blip_caption": "a photo of a desk with two monitors and a laptop", + "dia_id": "D27:19", + "text": "Yeah, I love it. It's like my own little haven to escape into the virtual world." + }, + { + "speaker": "Joanna", + "dia_id": "D27:20", + "text": "Wow, that sounds great to have your own gaming setup at home. It must be really awesome!" + }, + { + "speaker": "Nate", + "img_url": [ + "https://i.redd.it/h99h0hvw6x761.jpg" + ], + "blip_caption": "a photo of a pair of headphones and a video game controller", + "query": "gaming headset controller", + "dia_id": "D27:21", + "text": "It really is! But it's also really important to have something like this for my career, otherwise I would never be able to beat my competition." + }, + { + "speaker": "Joanna", + "dia_id": "D27:22", + "text": "That makes sense! It's all about practice isn't it? So what's your favorite game?" + }, + { + "speaker": "Nate", + "img_url": [ + "https://i.redd.it/7xcxqz1onyg91.jpg" + ], + "blip_caption": "a photo of two nintendo game covers with a picture of a group of people", + "query": "eternal kingdom game cover art", + "dia_id": "D27:23", + "text": "Yep! I'm currently playing this awesome fantasy RPG called \"Xeonoblade Chronicles\" and it's been a blast! I highly reccomend it if you've never played it before. " + }, + { + "speaker": "Joanna", + "dia_id": "D27:24", + "text": "What made you start playing it? That's a japanese game series right?" + }, + { + "speaker": "Nate", + "blip_caption": "a photo of a woman in a costume holding a glass", + "dia_id": "D27:25", + "text": "Yes it is! I'm a big fan of Nintendo games, and I've actually been wanting to play this one for a while because my friends have played it and reccomended it!" + }, + { + "speaker": "Joanna", + "blip_caption": "a photo of a captain america costume on display in a museum", + "dia_id": "D27:26", + "text": "Nice! It's really cool when a reccomendation from a friend fits your taste so well isn't it?" + }, + { + "speaker": "Nate", + "dia_id": "D27:27", + "text": "For sure! That's why I love when you give me movie reccomendations, I usually like them a lot more then if I were to just watch some random one." + }, + { + "speaker": "Joanna", + "img_url": [ + "https://i.redd.it/5hklfo3gjhx71.jpg" + ], + "blip_caption": "a photo of a handwritten letter from a young man", + "query": "handwritten letter movie script", + "dia_id": "D27:28", + "text": "Great to hear! I just finished with the intro to my next movie script, and I decided to include this at the begining." + }, + { + "speaker": "Nate", + "dia_id": "D27:29", + "text": "That letter is really awesome! Does it remind you of your childhood?" + }, + { + "speaker": "Joanna", + "dia_id": "D27:30", + "text": "Yeah, it does! My brother wrote it - he used to make me these cute notes when we were kids. Brings back sweet memories." + }, + { + "speaker": "Nate", + "dia_id": "D27:31", + "text": "Aww, childhood memories can be so powerful!" + }, + { + "speaker": "Joanna", + "blip_caption": "a photo of two little girls in pink dresses standing in front of a castle", + "dia_id": "D27:32", + "text": "They sure can! They take us back to simpler times but it's nice to create new memories as we grow up." + }, + { + "speaker": "Nate", + "dia_id": "D27:33", + "text": "Totally! I had a special day when I took my pets to the park. They were amazed and seeing their happy faces made it a memorable day. Mixing the new with the old is priceless - I treasure every memory!" + }, + { + "speaker": "Joanna", + "img_url": [ + "https://i.redd.it/tkkip0dr2eg91.jpg" + ], + "blip_caption": "a photo of a person holding a notebook with a list of things on it", + "query": "handwritten memories notebook", + "dia_id": "D27:34", + "text": "That sounds so sweet, Nate! I started writing some of my favorite memories down." + }, + { + "speaker": "Nate", + "dia_id": "D27:35", + "text": "Dang, your full of great ideas Joanna! I really should start doing that as well, or at least write down the things my animals like a lot!" + }, + { + "speaker": "Joanna", + "dia_id": "D27:36", + "text": "You should! I completely encourage it, looking back on fond memories is such a blessing." + }, + { + "speaker": "Nate", + "dia_id": "D27:37", + "text": "Ok I will! But I'll also start writing down some of my favorite memories with you from now on." + }, + { + "speaker": "Joanna", + "blip_caption": "a photo of two women sitting on a bed laughing and laughing", + "dia_id": "D27:38", + "text": "Definitely, let's keep making great memories and supporting each other. Let's keep reaching for our dreams and make them happen!" + } + ], + "session_28_date_time": "5:54 pm on 9 November, 2022", + "session_28": [ + { + "speaker": "Nate", + "img_url": [ + "https://beadyarnspatula.files.wordpress.com/2022/07/img_2596.jpg" + ], + "blip_caption": "a photo of a person scooping a scoop of ice cream into a pan", + "query": "coconut milk ice cream homemade red sprinkles", + "dia_id": "D28:1", + "text": "Hey Joanna, what a wild week! My game tournament got pushed back, so I tried out some cooking. Look at this homemade coconut ice cream! The sprinkles kinda changed the color this time around." + }, + { + "speaker": "Joanna", + "dia_id": "D28:2", + "text": "Hey Nate, that looks yummy! Wish I could try it, but I can't right now. How did the last game tournament go?" + }, + { + "speaker": "Nate", + "dia_id": "D28:3", + "text": "Hey Joanna, thanks! Tough tournament, didn't make it to the finals. But that's okay, I'll get 'em next time!" + }, + { + "speaker": "Joanna", + "dia_id": "D28:4", + "text": "Aww, bummer! But the important thing is to stay positive. So, what's your next move in the gaming world?" + }, + { + "speaker": "Nate", + "dia_id": "D28:5", + "text": "Thanks Joanna! Staying positive is key. I'm thinking of joining a new gaming team after this next tourney - I've had a few offers, but I haven't decided yet. It's gonna be a big step, but I'm ready for a shake up." + }, + { + "speaker": "Joanna", + "dia_id": "D28:6", + "text": "Sounds great, Nate! Making a switch could open up new opportunities. Wishing you luck with picking the right team!" + }, + { + "speaker": "Nate", + "dia_id": "D28:7", + "text": "Thanks, Joanna! I really appreciate it. It's a big decision, but I'm excited for what the future holds. How about you? Anything exciting happening on your end?" + }, + { + "speaker": "Joanna", + "dia_id": "D28:8", + "text": "Yup! I worked hard on another script and eventually created a plan for getting it made into a movie. It was a ton of work but satisfying. I pitched it to some producers yesterday and they really liked it. It gave me a big confidence boost!" + }, + { + "speaker": "Nate", + "blip_caption": "a photo of a trophy and a game controller on a table", + "dia_id": "D28:9", + "text": "Congrats on the chance to pitch your script - super impressive. Proud of you!" + }, + { + "speaker": "Joanna", + "img_url": [ + "https://www.penboutique.com/cdn/shop/products/image_d3480c08-098f-481b-a456-de0607a4fe91.jpg" + ], + "blip_caption": "a photo of a pen and notebook on a table with a book", + "query": "desk laptop notebook pens", + "dia_id": "D28:10", + "text": "Appreciate you, Nate! Your support and encouragement mean a lot to me. I feel like I just can't stop writing write now!" + }, + { + "speaker": "Nate", + "dia_id": "D28:11", + "text": "Anytime. What're you working on in that notebook? Anything cool?" + }, + { + "speaker": "Joanna", + "dia_id": "D28:12", + "text": "Hey Nate, I'm working on a new project - a suspenseful thriller set in a small Midwestern town. It's been a great creative outlet for me. How about you? Do you have any projects you're working on?" + }, + { + "speaker": "Nate", + "img_url": [ + "https://i.redd.it/yldt4paim5k71.jpg" + ], + "blip_caption": "a photo of a desk with a computer, headphones, and a microphone", + "query": "microphone headphones desk", + "dia_id": "D28:13", + "text": "Yeah actually - creating gaming content for YouTube. It's a cool way to entertain folks and satisfy my video game cravings at the same time when there aren't any tourneys going on." + }, + { + "speaker": "Joanna", + "dia_id": "D28:14", + "text": "Wow, that's a cool idea! What inspired you to start making gaming videos?" + }, + { + "speaker": "Nate", + "dia_id": "D28:15", + "text": "Hey Joanna, I'm a big fan of them and thought it would be a fun idea to start making them myself. I'm hoping to share my love of gaming and connect with others who enjoy it too." + }, + { + "speaker": "Joanna", + "dia_id": "D28:16", + "text": "Way to go, Nate! Making videos and connecting with people about gaming - that's awesome! You'll do great!" + }, + { + "speaker": "Nate", + "dia_id": "D28:17", + "text": "Thanks Joanna! Appreciate the support. It's new to me but I'm excited to get started!" + }, + { + "speaker": "Joanna", + "blip_caption": "a photo of a computer screen displaying a product listing", + "dia_id": "D28:18", + "text": "Make sure you watch other peoples videos first so you get a handle on what your audience likes! That way your videos don't flop when you post them." + }, + { + "speaker": "Nate", + "img_url": [ + "https://i.redd.it/i2l2lpmkj2mb1.jpg" + ], + "blip_caption": "a photo of a computer screen with a message on it", + "query": "positive comment youtube video screenshot", + "dia_id": "D28:19", + "text": "Already doing that, but thanks for the advice!" + }, + { + "speaker": "Joanna", + "img_url": [ + "https://i.redd.it/ap48tronzy411.jpg" + ], + "blip_caption": "a photo of a sunflower in a field with a sunset in the background", + "query": "sunset field flowers", + "dia_id": "D28:20", + "text": "No worries, Nate! It's great to support each other in reaching our goals. On another note, check out this pic I got a while back!" + }, + { + "speaker": "Nate", + "dia_id": "D28:21", + "text": "Wow, that sunset pic looks incredible! What inspired you to take that photo?" + }, + { + "speaker": "Joanna", + "dia_id": "D28:22", + "text": "Thanks, Nate! I took that pic on a hike last summer near Fort Wayne. The sunset and the surrounding beauty were just incredible. It was an awesome reminder of nature's beauty." + }, + { + "speaker": "Nate", + "img_url": [ + "https://i.redd.it/hgcgvd0umzsb1.jpg" + ], + "blip_caption": "a photo of three turtles sitting on a rock in a pond", + "query": "pet turtles basking sun", + "dia_id": "D28:23", + "text": "That sounds incredible! Nature truly has a way of reminding us to appreciate the beauty around us, and moments like those really stay with you. These critters also make me appreciate life's little joys. And guess what? I got them a new friend!" + }, + { + "speaker": "Joanna", + "dia_id": "D28:24", + "text": "Wow, what made you get a third?" + }, + { + "speaker": "Nate", + "blip_caption": "a photo of a turtle swimming in a tank with a metal bar", + "dia_id": "D28:25", + "text": "Turtles really bring me joy and peace. They have such an effect on us - best buddies ever! I saw another at a pet store and just hade to get him. The tank is big enough now for three, so I figured why not!" + }, + { + "speaker": "Joanna", + "dia_id": "D28:26", + "text": "Wow! It's always a shock where life will take us next! I bet just last week you would have never thought you would be getting a third turtle this year!" + }, + { + "speaker": "Nate", + "dia_id": "D28:27", + "text": "You got that right, but I'm very happy with the descision and wouldn't have it any other way." + }, + { + "speaker": "Joanna", + "dia_id": "D28:28", + "text": "Can I come over sometime and watch you play with them? From a distance I mean, since I'm allergic." + }, + { + "speaker": "Nate", + "dia_id": "D28:29", + "text": "Definitely! I'd love to have you over again. Maybe we can watch one of your movies together or go to the park!" + }, + { + "speaker": "Joanna", + "dia_id": "D28:30", + "text": "For sure! I'd love to do either of those things with you!" + }, + { + "speaker": "Nate", + "dia_id": "D28:31", + "text": "Sounds good. Well I'll make sure I give the turtles a bath before you get here so they're ready to play." + }, + { + "speaker": "Joanna", + "dia_id": "D28:32", + "text": "Alright, see you tomorrow!" + }, + { + "speaker": "Nate", + "dia_id": "D28:33", + "text": "Bye Joanna!" + } + ], + "session_29_date_time": "12:06 am on 11 November, 2022", + "session_29": [ + { + "speaker": "Joanna", + "img_url": [ + "https://i.redd.it/l7riz754xlj41.jpg" + ], + "blip_caption": "a photo of a person holding a clap board with a dog sleeping in a dog bed", + "query": "movie clapperboard movie set", + "dia_id": "D29:1", + "text": "Nate, can you believe it? I'm finally filming my own movie from the road-trip script!" + }, + { + "speaker": "Nate", + "blip_caption": "a photo of a trophy and a game controller on a table", + "dia_id": "D29:2", + "text": "Congrats, Joanna! Not surprised at all that your hard work paid off. Must feel awesome to see your script come alive in a movie! Pretty cool when something you love brings success, right? Tell me more about your movie!" + }, + { + "speaker": "Joanna", + "dia_id": "D29:3", + "text": "Woohoo, thanks Nate! It's pretty wild to see it come alive. Every day on set is awesome and full of potential. Being able to show my vision is awesome." + }, + { + "speaker": "Nate", + "blip_caption": "a photo of a fish tank with a fish inside of it", + "dia_id": "D29:4", + "text": "I think so too! What's been the coolest moment on set?" + }, + { + "speaker": "Joanna", + "dia_id": "D29:5", + "text": "One of the actors came up to me and told me how much she liked my script! I was so excited when that happened - it gave me chills!" + }, + { + "speaker": "Nate", + "blip_caption": "a photo of a turtle in a sink with a reflection of its head", + "dia_id": "D29:6", + "text": "Wow Joanna, that must have been so exciting! It's incredible when you get those moments of joy. Anyway, I took my turtles to the beach in Tampa yesterday! They always bring me peace in the craziness of life." + }, + { + "speaker": "Joanna", + "dia_id": "D29:7", + "text": "Woah, that's awesome, Nate! You must really enjoy having them around - they're so cool! What do you love most about having them?" + }, + { + "speaker": "Nate", + "img_url": [ + "https://i.redd.it/q35gmvlsr6yb1.jpg" + ], + "blip_caption": "a photo of a turtle sitting on a log in a pond", + "query": "pet turtles basking rock", + "dia_id": "D29:8", + "text": "Your completely right! I really love having them around. They're so cool and they make me feel calm. Plus, they don't require much looking after, which is great. I love seeing them soaking in the sun like this." + }, + { + "speaker": "Joanna", + "dia_id": "D29:9", + "text": "That's awesome, Nate! They look so serene and happy. It's great to have something like that." + }, + { + "speaker": "Nate", + "img_url": [ + "https://live.staticflickr.com/211/464853107_ccca166bec_b.jpg" + ], + "blip_caption": "a photography of a bowl of ice cream with a spoon in it", + "query": "homemade coconut milk ice cream colorful bowls", + "dia_id": "D29:10", + "re-download": true, + "text": "Yeah, turtles are like zen masters! They always remind me to slow down and appreciate the small things in life. I'm loving experimenting with flavors right now. Here are some colorful bowls of coconut milk ice cream that I made." + }, + { + "speaker": "Joanna", + "dia_id": "D29:11", + "text": "Hey Nate, that looks really yummy! The colors and mix-ins give it a nice kick." + }, + { + "speaker": "Nate", + "dia_id": "D29:12", + "text": "Nice! I'm glad you like it too. This recipe really jazzes it up. Wanna give it a try?" + }, + { + "speaker": "Joanna", + "dia_id": "D29:13", + "text": "Definitely, Nate! That ice cream looks mouthwatering. Thanks so much for offering!" + }, + { + "speaker": "Nate", + "dia_id": "D29:14", + "text": "No worries, Joanna. Hope you enjoy it!" + }, + { + "speaker": "Joanna", + "dia_id": "D29:15", + "text": "Yea, no worries! It was great catching up. Take it easy!" + } + ] + }, + "qa": [ + { + "question": "When did Joanna make a desert with almond milk?", + "answer": "The Friday before 14September, 2022", + "evidence": [ + "D21:9" + ], + "category": 2 + }, + { + "question": "What are Joanna's hobbies?", + "answer": "Writing, watchingmovies, exploringnature, hanging withfriends.", + "evidence": [ + "D1:10", + "D2:25" + ], + "category": 1 + }, + { + "question": "What is Joanna allergic to?", + "answer": "Most reptiles,animals with fur,cockroaches, dairy", + "evidence": [ + "D4:4", + "D5:11", + "D2:23" + ], + "category": 1 + }, + { + "question": "When did Nate win a big Valorant tourney?", + "answer": "The Saturday before 7November, 2022", + "evidence": [ + "D27:1" + ], + "category": 2 + }, + { + "question": "What type of movies does Nate enjoy watching the most?", + "answer": "action and sci-fi", + "evidence": [ + "D1:13" + ], + "category": 4 + }, + { + "question": "When did Nate go to a convention and meet new people?", + "answer": "The Friday before 9October, 2022.", + "evidence": [ + "D23:1" + ], + "category": 2 + }, + { + "question": "How many people attended the gaming party hosted by Nate in June 2022?", + "answer": "7", + "evidence": [ + "D16:6" + ], + "category": 4 + }, + { + "question": "What pets wouldn't cause any discomfort to Joanna?", + "answer": "Hairless cats or pigs,since they don't have fur, which is one of the main causes of Joanna's allergy.", + "evidence": [ + "D2:23" + ], + "category": 3 + }, + { + "question": "What alternative career might Nate consider after gaming?", + "answer": "an animalkeeper at a localzoo and workingwith turtles; as heknows a great dealabout turtles andhow to care for them,and he enjoys it.", + "evidence": [ + "D5:8", + "D19:3", + "D25:19", + "D28:25" + ], + "category": 3 + }, + { + "question": "What Console does Nate own?", + "answer": "A Nintendo Switch; since the game \"Xenoblade 2\" is made for this console.", + "evidence": [ + "D27:23" + ], + "category": 3 + }, + { + "question": "When did Nate adopt Max?", + "answer": "May 2022", + "evidence": [ + "D12:3" + ], + "category": 2 + }, + { + "question": "What state did Nate visit?", + "answer": "Florida", + "evidence": [ + "D29:6" + ], + "category": 3 + }, + { + "question": "When did Nate take time off to chill with his pets?", + "answer": "The weekend of 22August, 2022.", + "evidence": [ + "D19:9" + ], + "category": 2 + }, + { + "question": "When did Nate attend a cooking show?", + "answer": "The Monday before 14September, 2022", + "evidence": [ + "D21:4" + ], + "category": 2 + }, + { + "question": "What kind of interests do Joanna and Nate share?", + "answer": "Watching movies, making desserts", + "evidence": [ + "D1:10", + "D1:11", + "D1:12", + "D3:4", + "D4:9", + "D10:9", + "D20:2" + ], + "category": 1 + }, + { + "question": "What things has Nate reccomended to Joanna?", + "answer": "A pet,\"The Lord of the Rings\" movies,a dragon book series,coconut flavoring,\"Project Hail Mary\" book,Xenoblade Chronicles, dairy-free margarine, coconut oil", + "evidence": [ + "D2:14", + "D9:12", + "D9:14", + "D10:11", + "D19:17", + "D27:23", + "D10:19" + ], + "category": 1 + }, + { + "question": "What state did Joanna visit in summer 2021?", + "answer": "Indiana", + "evidence": [ + "D28:22" + ], + "category": 3 + }, + { + "question": "How many hikes has Joanna been on?", + "answer": "Four", + "evidence": [ + "D7:6", + "D11:5", + "D14:21", + "D28:22" + ], + "category": 3 + }, + { + "question": "What inspired Joanna's new script in July 2022?", + "answer": "Woodhaven's interesting past and people", + "evidence": [ + "D17:8" + ], + "category": 4 + }, + { + "question": "What is one of Nate's favorite dairy-free treats besides coconut milk ice cream?", + "answer": "dairy-free chocolate mousse", + "evidence": [ + "D21:10" + ], + "category": 4 + }, + { + "question": "What kind of films does Joanna enjoy?", + "answer": "Dramas and emotionally-driven films", + "evidence": [ + "D9:9" + ], + "category": 4 + }, + { + "question": "What kind of writings does Joanna do?", + "answer": "Screenplays,books, online blog posts, journal", + "evidence": [ + "D2:3", + "D17:14", + "D18:1", + "D18:5" + ], + "category": 1 + }, + { + "question": "What kind of books does Nate enjoy?", + "answer": "Adventures and magic", + "evidence": [ + "D9:14" + ], + "category": 4 + }, + { + "question": "How long did it take for Joanna to finish writing her book?", + "answer": "four months", + "evidence": [ + "D17:14", + "D22:9" + ], + "category": 2 + }, + { + "question": "How many turtles does Nate have?", + "answer": "Three", + "evidence": [ + "D8:3", + "D28:23" + ], + "category": 1 + }, + { + "question": "When did Nate win his first video game tournament?", + "answer": "the week before 21Janury, 2022", + "evidence": [ + "D1:3" + ], + "category": 2 + }, + { + "question": "When did Nate take his turtles to the beach?", + "answer": "10 November, 2022", + "evidence": [ + "D29:6" + ], + "category": 2 + }, + { + "question": "What recipes has Joanna made?", + "answer": "dairy free vanilla cake with strawberry filling and coconut cream frosting, parfait, strawberry chocolate cake, chocolate coconut cupcakes, chocolate raspberry tart, chocolate cake with raspberries, blueberry cheesecake bars", + "evidence": [ + "D10:9", + "D10:11", + "D19:8", + "D20:2", + "D20:10", + "D21:11", + "D22:1", + "D21:3", + "D21:17" + ], + "category": 1 + }, + { + "question": "Who was the new addition to Nate's family in May 2022?", + "answer": "Max", + "evidence": [ + "D12:3" + ], + "category": 2 + }, + { + "question": "When did Nate get his first two turtles?", + "answer": "2019", + "evidence": [ + "D2:12" + ], + "category": 2 + }, + { + "question": "Was the first half of September 2022 a good month career-wise for Nate and Joanna? Answer yes or no.", + "answer": "No; because both of them faced setbacks in their career", + "evidence": [ + "D20:1", + "D21:1" + ], + "category": 3 + }, + { + "question": "Which outdoor spot did Joanna visit in May?", + "answer": "Whispering Falls waterfall", + "evidence": [ + "D11:7" + ], + "category": 2 + }, + { + "question": "How many letters has Joanna recieved?", + "answer": "Two", + "evidence": [ + "D14:1", + "D18:5" + ], + "category": 1 + }, + { + "question": "What book recommendations has Joanna given to Nate?", + "answer": "\"Little Women\",'A Court of Thorns andRoses'.", + "evidence": [ + "D3:17", + "D19:14", + "D19:16" + ], + "category": 1 + }, + { + "question": "When did Nate win an international tournament?", + "answer": "21 August, 2022", + "evidence": [ + "D19:1" + ], + "category": 2 + }, + { + "question": "What advice does Joanna give to Nate about making YouTube videos?", + "answer": "Watch other people's videos to understand what the audience likes", + "evidence": [ + "D28:18" + ], + "category": 4 + }, + { + "question": "How did Joanna feel on October 25, 2022 about seeing her characters come alive on the big screen?", + "answer": "surreal and cool", + "evidence": [ + "D25:6" + ], + "category": 4 + } + ] + }, + { + "sample_id": "conv-43", + "conversation": { + "speaker_a": "Tim", + "speaker_b": "John", + "session_1_date_time": "7:48 pm on 21 May, 2023", + "session_1": [ + { + "speaker": "John", + "dia_id": "D1:1", + "text": "Hey Tim, nice to meet you! What's up? Anything new happening?" + }, + { + "speaker": "Tim", + "dia_id": "D1:2", + "text": "Hey John! Great to meet you. Been discussing collaborations for a Harry Potter fan project I am working on - super excited! Anything interesting happening for you?" + }, + { + "speaker": "John", + "dia_id": "D1:3", + "text": "That's great! I just signed with a new team - excited for the season!" + }, + { + "speaker": "Tim", + "dia_id": "D1:4", + "text": "Woohoo! Congrats on the new team. Which team did you sign with?" + }, + { + "speaker": "John", + "dia_id": "D1:5", + "text": "The Minnesota Wolves! I can't wait to play with them!" + }, + { + "speaker": "Tim", + "dia_id": "D1:6", + "text": "Cool! What position are you playing for the team? Any exciting games coming up?" + }, + { + "speaker": "John", + "img_url": [ + "https://i.redd.it/odlwr40mol581.jpg" + ], + "blip_caption": "a photo of a bunch of basketball jerseys laying on a bed", + "query": "basketball jersey collection", + "dia_id": "D1:7", + "text": "I'm a shooting guard for the team and our season opener is next week - so excited!" + }, + { + "speaker": "Tim", + "blip_caption": "a photo of a basketball game in progress with the ball in the air", + "dia_id": "D1:8", + "text": "Cool! Have any goals in mind?" + }, + { + "speaker": "John", + "img_url": [ + "https://blog.drdishbasketball.com/hubfs/IMG_0232%20%282%29.jpg" + ], + "blip_caption": "a photo of two men standing next to a basketball machine", + "query": "basketball hoop shooting percentage", + "dia_id": "D1:9", + "text": "Yeah, my goal is to improve my shooting percentage. Been practicing hard and gonna make it happen." + }, + { + "speaker": "Tim", + "dia_id": "D1:10", + "text": "Sounds good! What challenges have you encountered during your pre-season training?" + }, + { + "speaker": "John", + "dia_id": "D1:11", + "text": "Fitting into the new team's style of play was a challenge during pre-season." + }, + { + "speaker": "Tim", + "dia_id": "D1:12", + "text": "That sounds rough. How are things going with the new team?" + }, + { + "speaker": "John", + "blip_caption": "a photo of a bench in a locker with several lockers behind it", + "dia_id": "D1:13", + "text": "Things are going well! The team has been really nice and I'm having fun. How's your fan project coming along?" + }, + { + "speaker": "Tim", + "img_url": [ + "https://i.redd.it/l30ofyg973ja1.jpg" + ], + "blip_caption": "a photo of a table with a bunch of books on it", + "query": "harry potter books fan project collaboration wizarding world", + "dia_id": "D1:14", + "text": "It's been going well! Last week I talked to my friend who is a fan of Harry Potter and we're figuring out ideas, so it's been great to get lost in that magical world!" + }, + { + "speaker": "John", + "blip_caption": "a photo of a circle of shoes on the floor in a room", + "dia_id": "D1:15", + "text": "That's great! Loving it when people are passionate about their work. What kind of collaborations are you involved in for the fan project? I love talking to people about my sneaker collection." + }, + { + "speaker": "Tim", + "dia_id": "D1:16", + "text": "Thanks! We'll be discussing various aspects of the Harry Potter universe, like characters, spells, and magical creatures. It's great to see fans coming together for this." + }, + { + "speaker": "John", + "blip_caption": "a photo of a bookcase filled with books and toys", + "dia_id": "D1:17", + "text": "Wow! Have you been to any places related to it?" + }, + { + "speaker": "Tim", + "dia_id": "D1:18", + "text": "I went to a place in London a few years ago - it was like walking into a Harry Potter movie! I also went on a tour which was amazing. Have you been to any of the real Potter places? I'd love to explore them someday!" + }, + { + "speaker": "John", + "dia_id": "D1:19", + "text": "No, but it sounds fun! Going to those places is definitely on my to-do list." + }, + { + "speaker": "Tim", + "dia_id": "D1:20", + "text": "Definitely add it to your list! It's a really fun experience. Let me know if you need any tips for visiting. Bye!" + } + ], + "session_2_date_time": "5:08 pm on 15 June, 2023", + "session_2": [ + { + "speaker": "Tim", + "dia_id": "D2:1", + "text": "Last night I joined a fantasy literature forum and had a great talk about my fave books. It was so enriching!" + }, + { + "speaker": "John", + "dia_id": "D2:2", + "text": "Wow, great to hear that you had a blast talking books! It's cool to connect with others who share your passion. On a different note, exciting things are happening--I'm exploring endorsement opportunities. Thinking about the possibilities pumps me up. It would be amazing to work with brands and do something special. It's so rewarding to have my hard work pay off like this." + }, + { + "speaker": "Tim", + "dia_id": "D2:3", + "text": "Wow, that's awesome! Congrats - you must be so stoked! Which brands are you looking to link up with?" + }, + { + "speaker": "John", + "dia_id": "D2:4", + "text": "Thanks! I'm really excited about this new journey! I'm currently considering sports brands like Nike and Under Armour. It would be great to collaborate with brands that are related to sports. However, I'm also open to exploring other brands that align with my values and interests. There are so many options out there, and I can't wait to see where this takes me!" + }, + { + "speaker": "Tim", + "dia_id": "D2:5", + "text": "That's awesome! It's gotta be a rush having all these options. Can't wait to see which brands you choose to work with - gonna be great!" + }, + { + "speaker": "John", + "dia_id": "D2:6", + "text": "Thanks! Exciting times ahead! I'll keep you updated on which brands I choose. Can't wait to see where this journey leads me. Thanks for your support." + }, + { + "speaker": "Tim", + "img_url": [ + "https://i.redd.it/7m46xuo8lgj81.jpg" + ], + "blip_caption": "a photo of a book shelf with books and a picture on it", + "query": "bookshelf harry potter game of thrones", + "dia_id": "D2:7", + "text": "Yeah, John! Count on me for support. Can't wait to see what's up! This is my book collection so far." + }, + { + "speaker": "John", + "dia_id": "D2:8", + "text": "Wow, nice bookshelf! That picture is really interesting. What's up with it?" + }, + { + "speaker": "Tim", + "dia_id": "D2:9", + "text": "Thanks! That picture is from MinaLima. They created all the props for the Harry Potter films, and I love their work. It's like having a piece of the wizarding world at home!" + }, + { + "speaker": "John", + "dia_id": "D2:10", + "text": "Wow, having that is so cool! Your enthusiasm for it is awesome. You really go for it!" + }, + { + "speaker": "Tim", + "dia_id": "D2:11", + "text": "Thanks! I have lots of reminders of it - kind of a way to escape reality." + }, + { + "speaker": "John", + "dia_id": "D2:12", + "text": "Do those reminders help you escape the daily grind? Any chance you'll visit more places related to that world soon?" + }, + { + "speaker": "Tim", + "dia_id": "D2:13", + "text": "Definitely, those reminders really help. And there's definitely a chance I'll be visiting more HP spots in the future. It feels like I'm stepping into the books!" + }, + { + "speaker": "John", + "img_url": [ + "https://i.redd.it/fz2segua15d81.jpg" + ], + "blip_caption": "a photo of a basketball game being played in a gym", + "query": "basketball game", + "dia_id": "D2:14", + "text": "That sounds awesome! So cool that you get to immerse yourself in that world. So glad you found something that brings you so much joy. Keep diving in and enjoying it! Here's a pic from a recent game." + }, + { + "speaker": "Tim", + "dia_id": "D2:15", + "text": "Wow! That's awesome! Were you playing or watching?" + }, + { + "speaker": "John", + "dia_id": "D2:16", + "text": "Thanks! That was from a game I just played. I was in it! It was awesome being out there, doing what I love. Such an awesome feeling." + }, + { + "speaker": "Tim", + "dia_id": "D2:17", + "text": "Wow! You look so into it in that pic – it must be so awesome playing at that level! Keep rockin' it!" + }, + { + "speaker": "John", + "dia_id": "D2:18", + "text": "Thanks! It's a blast. Giving it my all every time I'm on the court. Really appreciate your support!" + }, + { + "speaker": "Tim", + "dia_id": "D2:19", + "text": "Yeah, keep going! Don't give up on your dreams. Talk to you later!" + } + ], + "session_3_date_time": "4:21 pm on 16 July, 2023", + "session_3": [ + { + "speaker": "John", + "img_url": [ + "https://live.staticflickr.com/44/145388126_0fd9b13887_b.jpg" + ], + "blip_caption": "a photography of a score board with a clock and a phone", + "query": "basketball scoreboard personal best", + "dia_id": "D3:1", + "re-download": true, + "text": "Hey Tim! Good to see you again. So much has happened in the last month - on and off the court. Last week I scored 40 points, my highest ever, and it feels like all my hard work's paying off." + }, + { + "speaker": "Tim", + "img_url": [ + "https://live.staticflickr.com/2174/2061292757_73e8ef5397_b.jpg" + ], + "blip_caption": "a photography of a table with a bunch of books on it", + "query": "harry potter books california discussion favorite books and characters magical experience", + "dia_id": "D3:2", + "re-download": true, + "text": "Congrats on your achievement! I'm so proud of you. Last week, I had a nice chat with a Harry Potter fan in California. It was magical!" + }, + { + "speaker": "John", + "img_url": [ + "https://i.redd.it/87ov530kw9ub1.jpg" + ], + "blip_caption": "a photo of a group of men sitting on top of a basketball court", + "query": "teammates celebrating court", + "dia_id": "D3:3", + "text": "Thank you! Scoring those points was an incredible experience. The atmosphere was electric, and my teammates and I were thrilled. We pulled off a tough win!" + }, + { + "speaker": "Tim", + "dia_id": "D3:4", + "text": "Wow, sounds awesome! Winning after that game must have felt amazing - what was it like? Did you celebrate afterward?" + }, + { + "speaker": "John", + "blip_caption": "a photo of a group of people sitting at a table eating", + "dia_id": "D3:5", + "text": "We were all exhausted but so happy. After that, we celebrated at a restaurant, laughing and reliving the intense moments - it felt amazing!" + }, + { + "speaker": "Tim", + "dia_id": "D3:6", + "text": "Wow, sounds like a blast! I had an incredible time meeting with that fellow fan. You can really feel the love when you're surrounded by people who share the same passion. Does that happen with your sport too?" + }, + { + "speaker": "John", + "blip_caption": "a photo of a rack of basketball jerseys in a store", + "dia_id": "D3:7", + "text": "Definitely! Being surrounded by teammates who are equally passionate creates a strong bond. We push each other to be our best and the love for the game is infectious. It's like having a second family." + }, + { + "speaker": "Tim", + "dia_id": "D3:8", + "text": "That's awesome! Having a second family through sport must be such a great feeling. Glad you have that support. Oh, you mentioned exploring endorsements - have you made any progress?" + }, + { + "speaker": "John", + "img_url": [ + "https://i.imgur.com/uvVOoeM.jpg" + ], + "blip_caption": "a photo of a handwritten letter with a black ink marker", + "query": "handwritten marketing plan", + "dia_id": "D3:9", + "text": "Yeah, I'm getting somewhere with endorsements. I've talked to some big names, which looks promising. Exciting to see what's in store!" + }, + { + "speaker": "Tim", + "dia_id": "D3:10", + "text": "How did you manage to connect with these big companies?" + }, + { + "speaker": "John", + "blip_caption": "a photo of a basketball card with a picture of a man holding a basketball", + "dia_id": "D3:11", + "text": "I used my contacts in the basketball industry and my marketing skills to make connections. Networking plays a big role in getting endorsements, and I'm grateful for the support I've received." + }, + { + "speaker": "Tim", + "dia_id": "D3:12", + "text": "Wow, what endorsements have you managed to get through networking?" + }, + { + "speaker": "John", + "dia_id": "D3:13", + "text": "I just signed up Nike for a basketball shoe and gear deal. I'm also in talks with Gatorade about a potential sponsorship. It's pretty cool to be working with such big brands!" + }, + { + "speaker": "Tim", + "dia_id": "D3:14", + "text": "Wow, Congrats on those deals with Nike and Gatorade! You're killing it! Any other brands you're dreaming of working with?" + }, + { + "speaker": "John", + "img_url": [ + "https://hips.hearstapps.com/hmg-prod/images/img-8577-1571255920.jpg" + ], + "blip_caption": "a photo of a mannequin in a blue suit and a chair", + "query": "under armour advertisement", + "dia_id": "D3:15", + "text": "Thanks! The Nike and Gatorade deals have me stoked! I've always liked Under Armour, working with them would be really cool." + }, + { + "speaker": "Tim", + "dia_id": "D3:16", + "text": "Wow! What kind of stuff are you exploring? It looks like good things are coming your way." + }, + { + "speaker": "John", + "img_url": [ + "https://images.pexels.com/photos/5854539/pexels-photo-5854539.jpeg?cs\\u003dsrgb\\u0026dl\\u003dpexels-millah-5854539.jpg" + ], + "blip_caption": "a photo of a city skyline at sunset with a body of water", + "query": "city skyline sunset", + "dia_id": "D3:17", + "text": "Just checking out some exciting things that are happening. Really looking forward to what's coming next! This is where I'm headed." + }, + { + "speaker": "Tim", + "dia_id": "D3:18", + "text": "Wow, amazing view! Where's that? What's got you so excited?" + }, + { + "speaker": "John", + "img_url": [ + "https://i.redd.it/ua7cvodhrkca1.jpg" + ], + "blip_caption": "a photo of a crowd of people watching a basketball game", + "query": "seattle basketball court bright lights", + "dia_id": "D3:19", + "text": "It's Seattle, I'm stoked for my game there next month! It's one of my favorite cities to explore - super vibrant!" + }, + { + "speaker": "Tim", + "dia_id": "D3:20", + "text": "Cool! What do you love about Seattle?" + }, + { + "speaker": "John", + "dia_id": "D3:21", + "text": "I love the energy, diversity, and awesome food of this city. Trying local seafood is a must! Plus, the support from the fans at games is incredible." + }, + { + "speaker": "Tim", + "blip_caption": "a photo of a stack of three plates of food with crab legs", + "dia_id": "D3:22", + "text": "Sounds fab! Seattle is definitely a great and colorful city. I've always wanted to try the seafood there. Good luck with everything!" + }, + { + "speaker": "John", + "img_url": [ + "https://i.pinimg.com/originals/c8/d6/bc/c8d6bc6ad3b08172bdbc1665f3ed080b.jpg" + ], + "blip_caption": "a photo of a person walking on the beach with a surfboard", + "query": "sunset ocean", + "dia_id": "D3:23", + "text": "Thanks! Can't wait for the seafood too. I love the ocean." + }, + { + "speaker": "Tim", + "dia_id": "D3:24", + "text": "That looks peaceful! Do you have a favorite beach memory?" + }, + { + "speaker": "John", + "img_url": [ + "https://images.pexels.com/photos/8890340/pexels-photo-8890340.jpeg?cs\\u003dsrgb\\u0026dl\\u003dpexels-anastasia-baranova-8890340.jpg" + ], + "blip_caption": "a photo of a man holding a surfboard on a beach", + "query": "surfing waves beach scenery surfboard", + "dia_id": "D3:25", + "text": "I had an awesome summer with my friends, surfing and riding the waves. The feeling was unreal!" + }, + { + "speaker": "Tim", + "dia_id": "D3:26", + "text": "Wow! How long have you been surfing?" + }, + { + "speaker": "John", + "dia_id": "D3:27", + "text": "I started surfing five years ago and it's been great. I love the connection to nature." + }, + { + "speaker": "Tim", + "blip_caption": "a photo of a person riding a surfboard on a body of water", + "dia_id": "D3:28", + "text": "Wow! That sounds amazing! The connection to nature must be incredible." + }, + { + "speaker": "John", + "img_url": [ + "https://i.pinimg.com/originals/c8/d6/bc/c8d6bc6ad3b08172bdbc1665f3ed080b.jpg" + ], + "blip_caption": "a photo of a person walking on the beach with a surfboard", + "query": "sunset ocean", + "dia_id": "D3:29", + "text": "Yup! Being out in the water can be amazing. The waves, the wind, it's super exciting and free-feeling. Nature's pretty special." + }, + { + "speaker": "Tim", + "img_url": [ + "https://d3525k1ryd2155.cloudfront.net/h/072/089/1318089072.0.x.jpg" + ], + "blip_caption": "a photo of a book with a harry potter cover", + "query": "harry potter books", + "dia_id": "D3:30", + "text": "That's awesome! I don't surf, but reading a great fantasy book helps me escape and feel free." + }, + { + "speaker": "John", + "dia_id": "D3:31", + "text": "Cool! We all find our own way to escape and feel free!" + }, + { + "speaker": "Tim", + "img_url": [ + "https://cdn.apartmenttherapy.info/image/upload/v1617734023/at/news-culture/2021-04/katharine-scrivener-nook.jpg" + ], + "blip_caption": "a photo of a living room with a brown couch and a white ottoman", + "query": "cozy reading nook blanket pillows", + "dia_id": "D3:32", + "text": "Yeah! It's great to find stuff that makes us happy and feel free. It's like bliss for me when I do this in a comfy spot. It's like being in another world, same as surfing is for you." + }, + { + "speaker": "John", + "dia_id": "D3:33", + "text": "Yeah! Those moments of happiness and freedom are amazing. Let's all find our own bliss." + }, + { + "speaker": "Tim", + "dia_id": "D3:34", + "text": "Sure thing! It's what makes life awesome!" + }, + { + "speaker": "John", + "dia_id": "D3:35", + "text": "Yeah. Awesome catching up! Bye!" + } + ], + "session_4_date_time": "4:17 pm on 2 August, 2023", + "session_4": [ + { + "speaker": "Tim", + "dia_id": "D4:1", + "text": "Hey John! How've you been? Something awesome happened - I'm writing articles about fantasy novels for an online mag. It's so rewarding!" + }, + { + "speaker": "John", + "dia_id": "D4:2", + "text": "Hey Tim! Congrats on the opportunity to write about what you're into! How did it happen?" + }, + { + "speaker": "Tim", + "dia_id": "D4:3", + "text": "Thanks! I found this opportunity on a fantasy lit forum and thought it'd be perfect since I love fantasy. I shared my ideas with the magazine and they liked them! It's been awesome to spread my love of fantasy." + }, + { + "speaker": "John", + "dia_id": "D4:4", + "text": "Congratulations! That's awesome. What kind of articles have you been writing?" + }, + { + "speaker": "Tim", + "dia_id": "D4:5", + "text": "Thanks! I've been writing about different fantasy novels, studying characters, themes, and making book recommendations." + }, + { + "speaker": "John", + "dia_id": "D4:6", + "text": "Awesome! Must be so rewarding to delve into your books and chat about them. Do you have any favorite books you love writing about?" + }, + { + "speaker": "Tim", + "dia_id": "D4:7", + "text": "For sure! Harry Potter and Game of Thrones are amazing - I'm totally hooked! I could chat about them forever!" + }, + { + "speaker": "John", + "img_url": [ + "https://pictures.abebooks.com/inventory/31262885157_3.jpg" + ], + "blip_caption": "a photo of a book with a picture of a person holding a bookmark", + "query": "harry potter signed book", + "dia_id": "D4:8", + "text": "Oh yeah, I remember you telling me about Harry Potter! I've got a funny story. Anthony and I went to this charity thing and ended up in this intense Harry Potter trivia contest. We did alright, but there was this one super-nerd there that took home this as a prize.\n" + }, + { + "speaker": "Tim", + "dia_id": "D4:9", + "text": "That looks great! The signature is sweet! Have you been reading anything?" + }, + { + "speaker": "John", + "dia_id": "D4:10", + "text": "I've been reading this inspiring book, it reminds me to keep dreaming." + }, + { + "speaker": "Tim", + "dia_id": "D4:11", + "text": "Books can really inspire and help us keep our dreams alive. Keep it up!" + }, + { + "speaker": "John", + "dia_id": "D4:12", + "text": "Thanks! They really do. I want to keep reaching for new goals." + }, + { + "speaker": "Tim", + "dia_id": "D4:13", + "text": "Same here!" + }, + { + "speaker": "John", + "dia_id": "D4:14", + "text": "Have fun with your writing! Catch you later!" + }, + { + "speaker": "Tim", + "dia_id": "D4:15", + "text": "Thanks! I'll enjoy writing them. Take care and talk soon!" + } + ], + "session_5_date_time": "10:29 am on 9 August, 2023", + "session_5": [ + { + "speaker": "Tim", + "dia_id": "D5:1", + "text": "Hey John! Long time no see! Been super busy lately. Guess what? Just skyped with that Harry Potter fan I met in CA and had a great time. We talked characters and maybe collab-ing - so cool to talk to someone who gets it. You? Anything new going on?" + }, + { + "speaker": "John", + "img_url": [ + "https://i0.wp.com/thepirateshook.com/wp-content/uploads/2022/11/img_7438-edited.jpg" + ], + "blip_caption": "a photo of a basketball game being played in a large arena", + "query": "intense game basketball winning shot crowd", + "dia_id": "D5:2", + "text": "Hi Tim! Nice to hear from you. Glad you could reconnect. As for me, lots of stuff happened since we last talked. Last week I had a crazy game - crazy intense! We won it by a tight score. Scoring that last basket and hearing the crowd cheer was awesome!" + }, + { + "speaker": "Tim", + "dia_id": "D5:3", + "text": "Nice work! Bet it felt awesome to score that basket and have the crowd going wild. Must have been such an adrenaline rush! Did you manage to capture any other photos from the game?" + }, + { + "speaker": "John", + "dia_id": "D5:4", + "text": "Thanks! It was an amazing rush and just the one I showed you. We were so hyped!" + }, + { + "speaker": "Tim", + "dia_id": "D5:5", + "text": "Awesome! Winning a tough game must have been such an exhilarating experience!" + }, + { + "speaker": "John", + "dia_id": "D5:6", + "text": "Our team bond is awesome and it makes all the hard work worth it." + }, + { + "speaker": "Tim", + "dia_id": "D5:7", + "text": "It's incredible how a team creates such strong ties. Having support like that is so important." + }, + { + "speaker": "John", + "dia_id": "D5:8", + "text": "Thanks! You nailed it! Having a strong team/support is key - it's like a family away from home. We push each other to improve, and I'm so thankful for them." + }, + { + "speaker": "Tim", + "dia_id": "D5:9", + "text": "That's great! Having a supportive team who are like family is awesome. Having people who motivate you and stick by you is priceless." + }, + { + "speaker": "John", + "dia_id": "D5:10", + "text": "Definitely! They encourage me when I'm down too. It's not just in my sport, but in other aspects of life too. We hang out a lot and it's great having that bond." + }, + { + "speaker": "Tim", + "dia_id": "D5:11", + "text": "Yeah, having another family is great. It definitely helps with my home life and hobbies." + }, + { + "speaker": "John", + "dia_id": "D5:12", + "text": "Having someone to support and motivate you is so important, whether it's in sports or any other aspect of life. I know you've found your peace in reading fantasy books - that's amazing! What book are you currently reading? Anything that has stood out to you?" + }, + { + "speaker": "Tim", + "dia_id": "D5:13", + "text": "Thanks for asking! I'm reading a fantasy book that really captivates me. It takes me to another world where I'm on the edge of my seat and my imagination soars. It's amazing how books can transport us like that." + }, + { + "speaker": "John", + "dia_id": "D5:14", + "text": "Books can be so captivating, taking us on such incredible journeys! What's the name of it?" + }, + { + "speaker": "Tim", + "dia_id": "D5:15", + "text": "It's a book by Patrick Rothfuss and it's awesome! The way the author builds the world and characters is amazing. You should read it!" + }, + { + "speaker": "John", + "dia_id": "D5:16", + "text": "Sounds cool! I'll definitely check it out. Thanks for the recommendation!" + }, + { + "speaker": "Tim", + "dia_id": "D5:17", + "text": "No problem! Let me know what you think after you read it." + }, + { + "speaker": "John", + "dia_id": "D5:18", + "text": "Yep, I'll let you know once I'm done reading it. Thanks!" + }, + { + "speaker": "Tim", + "dia_id": "D5:19", + "text": "I hope you like it. Chat soon!" + }, + { + "speaker": "John", + "dia_id": "D5:20", + "text": "Me too. Talk to you soon!" + } + ], + "session_6_date_time": "1:08 pm on 11 August, 2023", + "session_6": [ + { + "speaker": "John", + "dia_id": "D6:1", + "text": "Hey Tim, sorry I missed you. Been a crazy few days. Took a trip to a new place - it's been amazing. Love the energy there." + }, + { + "speaker": "Tim", + "dia_id": "D6:2", + "text": "Hey John, no worries! I get how life can be busy. Where did you go? Glad you had a great time! Exploring new places can be so inspiring and fun. I recently went to an event and it was fantastic. Being with other fans who love it too was so special. Have you ever gone to an event related to something you like?" + }, + { + "speaker": "John", + "dia_id": "D6:3", + "text": "I was in Chicago, it was awesome! It had so much energy and the locals were really friendly. It's great to experience other cultures and connect with new folks." + }, + { + "speaker": "Tim", + "dia_id": "D6:4", + "text": "Wow, Chicago sounds great! It's refreshing to try something new and connect with people from different backgrounds. Have you ever been to a sports game and felt a real connection with the other fans?" + }, + { + "speaker": "John", + "dia_id": "D6:5", + "text": "Yeah! There's nothing like the energy in a stadium during a game. Everyone's cheering, chanting, and getting so excited. It's a really special experience!" + }, + { + "speaker": "Tim", + "img_url": [ + "https://cdn27.picryl.com/photo/1934/01/01/herbert-brutus-ehrmann-papers-1906-1970-sacco-vanzetti-book-review-by-edmund-216c54-1024.jpg" + ], + "blip_caption": "a photography of a book opened to a page with a picture of a man", + "query": "fantasy novel article screenshot", + "dia_id": "D6:6", + "re-download": true, + "text": "I can just imagine the thrill of being in that kind of atmosphere. Must've been an amazing experience for you! BTW, I have been writing more articles - it lets me combine my love for reading and the joy of sharing great stories. Here's my latest one!" + }, + { + "speaker": "John", + "dia_id": "D6:7", + "text": "That's awesome! Have you come across any interesting books lately?" + }, + { + "speaker": "Tim", + "img_url": [ + "https://i.redd.it/ydnhc3r3efw61.jpg" + ], + "blip_caption": "a photo of a book set of three books on a wooden table", + "query": "name of the wind patrick rothfuss book cover", + "dia_id": "D6:8", + "text": "Thanks! \"The Name of the Wind\" is great. It's a fantasy novel with a great magician and musician protagonist. The world-building and character development are really good. Definitely worth a read if you're looking for something captivating!" + }, + { + "speaker": "John", + "img_url": [ + "https://images.novelship.com/product-lookbook/ao7VLb2K1Qnp-1682611652416-54349732-76B9-483E-B2C2-CE8033DA879E.jpg" + ], + "blip_caption": "a photo of a pair of sneakers in a box", + "query": "lucky basketball shoes", + "dia_id": "D6:9", + "text": "That book sounds awesome! Love a good fantasy with strong characters and cool world-building. Cheers for the suggestion. Adding it to my list. These are my lucky basketball shoes. They've been with me through the good and bad. Every mark has a story." + }, + { + "speaker": "Tim", + "dia_id": "D6:10", + "text": "Your shoes must have a lot of stories behind them. Want to share some with me?" + }, + { + "speaker": "John", + "dia_id": "D6:11", + "text": "Yes, these have been with me on my journey since the beginning. All the successes, the failures, the friends - I have so many stories to tell. They're more than just a pair of shoes, they symbolize resilience, determination, and a love for the game. They remind me of what I've achieved and how far I've come." + }, + { + "speaker": "Tim", + "dia_id": "D6:12", + "text": "Those shoes are special. They show your hard work, your successes, and all the amazing times you've had with basketball. It's awesome how meaningful objects can become. So inspiring. How did you get into the game?" + }, + { + "speaker": "John", + "dia_id": "D6:13", + "text": "Thanks! Basketball has been a part of my life ever since I was a kid. I'd watch NBA games with my dad and dream of playing on those big courts. When I turned ten, dad signed me up for a local league, and I've been playing ever since. I kept playing through middle and high school before earning a college scholarship. And after college, I was drafted by a team – my dream come true!" + }, + { + "speaker": "Tim", + "dia_id": "D6:14", + "text": "Wow! You really made your childhood dream come true. It's impressive how your dedication and hard work paid off. It's awesome how our passions shape our lives. Do you have any big goals for your basketball career?" + }, + { + "speaker": "John", + "dia_id": "D6:15", + "text": "Yeah! Winning a championship is my number one goal. But I also want to make a difference away from the court, like through charity or inspiring people. Basketball has been great to me, so I want to give something back." + }, + { + "speaker": "Tim", + "dia_id": "D6:16", + "text": "Winning a title and making a difference off the court is inspiring. How do you plan to kick off your charity work?" + }, + { + "speaker": "John", + "dia_id": "D6:17", + "text": "I'm teaming up with a local organization that helps disadvantaged kids with sports and school. I'm hoping to use my platform to have a positive impact on the community and inspire others as well." + }, + { + "speaker": "Tim", + "dia_id": "D6:18", + "text": "Making a difference like that is truly amazing. I can't wait to see the impact it'll have. All the best for your charity work!" + }, + { + "speaker": "John", + "dia_id": "D6:19", + "text": "Thanks! Really appreciate the support. It means a lot. I'm excited to work hard and make a positive impact." + }, + { + "speaker": "Tim", + "dia_id": "D6:20", + "text": "No worries. I'm here to support you. You've got tons of determination and passion! Keep it up - you're gonna make a difference!" + }, + { + "speaker": "John", + "dia_id": "D6:21", + "text": "Thanks! Your words mean a lot. I'll do my best!" + }, + { + "speaker": "Tim", + "dia_id": "D6:22", + "text": "Glad I could help. You've got this!" + }, + { + "speaker": "John", + "dia_id": "D6:23", + "text": "Thanks! Talk to you later!" + } + ], + "session_7_date_time": "7:54 pm on 17 August, 2023", + "session_7": [ + { + "speaker": "John", + "dia_id": "D7:1", + "text": "Hey Tim! We had a wild few days since we talked. I met back up with my teammates on the 15th after my trip and it was amazing! Everyone missed me. The atmosphere was electric and I felt so welcome being back with them. I'm so lucky to be a part of this team!" + }, + { + "speaker": "Tim", + "dia_id": "D7:2", + "text": "Wow, John, that sounds amazing! I'm so happy they gave you a warm welcome back. It's such a special feeling when you realize that you share the same passions and talents with others. It's like finding your true place in the world." + }, + { + "speaker": "John", + "dia_id": "D7:3", + "text": "Thanks! Wow, it was such an incredible experience. Being around people who share the same love for basketball creates a special kind of bond. Have you ever felt like you truly belonged somewhere?" + }, + { + "speaker": "Tim", + "dia_id": "D7:4", + "text": "Yeah, definitely. I felt like I belonged a few times, but last month at that event was one of my favorites. Everyone shared the same love for it and it felt like being in a world where everyone understood it. I'm really thankful for those experiences - it's great to know there are people out there who appreciate and share my interests." + }, + { + "speaker": "John", + "dia_id": "D7:5", + "text": "Cool! It's great when you find a group that appreciates the same things. It really adds something special to our lives. So, do you have any exciting plans or events coming up?" + }, + { + "speaker": "Tim", + "dia_id": "D7:6", + "text": "I have no big events coming up, but I'm hoping to attend a book conference next month. It's an interesting gathering of authors, publishers and book lovers where we talk about our favorite novels and new releases. I'm excited to go because it'll help me learn more about literature and create a stronger bond to it." + }, + { + "speaker": "John", + "img_url": [ + "https://upload.wikimedia.org/wikipedia/commons/6/61/2018_DII_Elite_Eight_Northern_State_Signed_Basketball.jpg" + ], + "blip_caption": "a photography of a basketball with autographs on it sitting on a table", + "query": "basketball signed teammates", + "dia_id": "D7:7", + "re-download": true, + "text": "You're a real bookworm! It would be awesome to go to a book conference with you. Check out this photo of what my teammates gave me when we met. It's a sign of our friendship and all the love we have for each other." + }, + { + "speaker": "Tim", + "dia_id": "D7:8", + "text": "That's so cool of your teammates. Did they sign it for a special reason?" + }, + { + "speaker": "John", + "dia_id": "D7:9", + "text": "Thanks! They signed it to show our friendship and appreciation. It's a great reminder of our bond." + }, + { + "speaker": "Tim", + "dia_id": "D7:10", + "text": "That's really cool. It's great that you have something to remind you of your friends. Keeping a bit of their energy and support with you is always nice." + }, + { + "speaker": "John", + "dia_id": "D7:11", + "text": "Having something like this ball to remind me of the bond and support from my teammates is really comforting. It's a nice reminder of why I started playing basketball and my journey. It motivates me to stay strong and give it my all." + }, + { + "speaker": "Tim", + "dia_id": "D7:12", + "text": "That's so sweet. It's great to have something so meaningful to keep you motivated. I'll keep that in mind next time I need a push to reach my goals." + }, + { + "speaker": "John", + "dia_id": "D7:13", + "text": "It's really motivating to have something that reminds you of why you started, and having supportive people around is like having a cheer team that helps you through tough times." + }, + { + "speaker": "Tim", + "dia_id": "D7:14", + "text": "Yeah, that's true. Having them there to cheer you on can be a powerful source of strength." + }, + { + "speaker": "John", + "dia_id": "D7:15", + "text": "Yeah, having that support really encourages me to give it my all and never give up. It's an awesome feeling!" + }, + { + "speaker": "Tim", + "dia_id": "D7:16", + "text": "It's awesome how much strength people can get from each other. Bye!" + } + ], + "session_8_date_time": "4:29 pm on 21 August, 2023", + "session_8": [ + { + "speaker": "John", + "img_url": [ + "https://d2rzw8waxoxhv2.cloudfront.net/facilities/xlarge/fafaccd30c88b7506920/1569529088544-992-69.jpg" + ], + "blip_caption": "a photo of a gym with a basketball court and cones", + "query": "basketball court gym", + "dia_id": "D8:1", + "text": "Hey Tim! Long time no talk. Hope you're doing great. Crazy things have been going on in my life. Just the other day, I found a new gym to stay on my b-ball game. Staying fit is essential to surviving pro ball, so I had to find something that fits the bill. Finding the right spot was tough but here we are!" + }, + { + "speaker": "Tim", + "dia_id": "D8:2", + "text": "Hey John! Really good to hear from you. Staying fit is so important. Must be so cool to practice there. Any issues you had when you got it?" + }, + { + "speaker": "John", + "dia_id": "D8:3", + "text": "It's been great training here. The gym is awesome, but I had to overcome the hurdle of adapting and tweaking my routine. Finding the right balance was tricky, but I eventually got the hang of it." + }, + { + "speaker": "Tim", + "dia_id": "D8:4", + "text": "Nice one! It can be tough getting used to a new routine, but once you figure it out, it gets easier. How did you find that balance?" + }, + { + "speaker": "John", + "img_url": [ + "https://i.redd.it/sch5pby1ivi31.jpg" + ], + "blip_caption": "a photo of a notebook with a list of items on it", + "query": "workout schedule", + "dia_id": "D8:5", + "text": "Thanks! Took some trial and error but I figured out a schedule with both basketball stuff and strength training to balance it out. Listening to my body and giving it enough rest made it easier to push myself during practice but also look after me. Here's my workout plan. It helps a lot with staying on track." + }, + { + "speaker": "Tim", + "dia_id": "D8:6", + "text": "Nice job! Impressive plan you've got there! You've really thought it out. Why include strength training in your routine?" + }, + { + "speaker": "John", + "dia_id": "D8:7", + "text": "Thanks! Strength training is important for basketball because it builds muscle, increases power, and prevents injuries. It also helps me become more explosive, which is essential in games. Plus, it boosts my athleticism overall." + }, + { + "speaker": "Tim", + "dia_id": "D8:8", + "text": "That makes sense! Your holistic approach seems to have numerous benefits. Does strength training have a positive impact on your basketball performance?" + }, + { + "speaker": "John", + "dia_id": "D8:9", + "text": "Definitely! Incorporating strength training really changed the game for me, improving my shooting accuracy, agility, and speed. It gave me the upper hand over my opponents and helped me up my game. It gave me the confidence to take on whatever comes my way." + }, + { + "speaker": "Tim", + "dia_id": "D8:10", + "text": "Awesome! Gaining confidence on the court must feel great. It's cool how strength training can benefit you. You're doing great in both basketball and fitness, keep it up!" + }, + { + "speaker": "John", + "dia_id": "D8:11", + "text": "Thanks! Appreciate your support. It's been a journey, but I'm happy with the progress. Excited to see what's next. What about you? How have you been?" + }, + { + "speaker": "Tim", + "dia_id": "D8:12", + "text": "Things have been great since we last talked - I've been focusing on school and reading a bunch of fantasy books. It's a nice way to take a break from all the stress. I've also started learning how to play the piano - it's a learning curve, but it's so satisfying seeing the progress I make! Life's good." + }, + { + "speaker": "John", + "dia_id": "D8:13", + "text": "Wow! You're staying busy and having fun. Learning to play this is awesome - it's such a beautiful instrument. Do you have any favorite songs you like playing on it?" + }, + { + "speaker": "Tim", + "dia_id": "D8:14", + "text": "Thanks! I love playing different songs on the piano, but my favorite one to jam to is a theme from a movie I really enjoy. It brings back lots of great memories." + }, + { + "speaker": "John", + "dia_id": "D8:15", + "text": "Wow, that's cool! Music really has a way of bringing back memories and evoking emotions, doesn't it? Almost like taking us back in time. Could you tell me more about that film and the memories it brings up for you?" + }, + { + "speaker": "Tim", + "dia_id": "D8:16", + "text": "Yeah, \"Harry Potter and the Philosopher's Stone\" is special to me. It was the first movie from the series and brings back some great memories. Watching it with my family was amazing. It was so magical!" + }, + { + "speaker": "John", + "img_url": [ + "https://www.goodwillfinds.com/on/demandware.static/-/Sites-goodwill-master/default/dw41d27013/images/large/lhyOBm1CPSKy54szJay7vQj/2023/November/14/image_(140).jpg" + ], + "blip_caption": "a photo of a dvd cover with a castle in the background", + "query": "harry potter dvd collection", + "dia_id": "D8:17", + "text": "Wow, that sounds great, Tim! I love that first movie too, I even have the whole collection! It was so magical! Must've been a dream watching it with your family." + }, + { + "speaker": "Tim", + "dia_id": "D8:18", + "text": "It was really a dream come true! Watching that movie with my family was awesome, we'd all get comfy with snacks and a blanket and be totally absorbed. Such a special memory!" + }, + { + "speaker": "John", + "blip_caption": "a photo of a group of people standing around a kitchen table", + "dia_id": "D8:19", + "text": "Cool! Cherish those family moments - they're so irreplaceable. Family time is great! Mine gets together all the time too." + }, + { + "speaker": "Tim", + "img_url": [ + "https://i.redd.it/peqj8rke7zz11.jpg" + ], + "blip_caption": "a photo of a family sitting on a couch in front of a fireplace", + "query": "Thanksgiving family photo", + "dia_id": "D8:20", + "text": "Family time means a lot to me. This photo is from a special day when we all got together to eat. It was a great day full of love and laughter!" + }, + { + "speaker": "John", + "dia_id": "D8:21", + "text": "Wow, that looks like such a great day! Do you have any favorite Thanksgiving traditions?" + }, + { + "speaker": "Tim", + "dia_id": "D8:22", + "text": "Thanksgiving's always special for us. We love prepping the feast and talking about what we're thankful for. Plus, watching some movies afterwards - the best!" + }, + { + "speaker": "John", + "dia_id": "D8:23", + "text": "Thanksgiving dinner with family sounds great! Do you have any favorite movies you watch together?" + }, + { + "speaker": "Tim", + "img_url": [ + "https://i.redd.it/4zp9nizyheq61.jpg" + ], + "blip_caption": "a photo of a dvd cover with a child in a house", + "query": "home alone movie poster", + "dia_id": "D8:24", + "text": "During Thanksgiving, we usually watch a few movies. We love \"Home Alone\" - it always brings lots of laughs!" + }, + { + "speaker": "John", + "dia_id": "D8:25", + "text": "That's a classic! What other movies do you watch during the holidays?" + }, + { + "speaker": "Tim", + "blip_caption": "a photo of a dvd cover of a movie with a leprechaun", + "dia_id": "D8:26", + "text": "We also watch \"Elf\" during the holidays. It makes us laugh and get us feeling festive!" + }, + { + "speaker": "John", + "dia_id": "D8:27", + "text": "Those are awesome! Any other holiday movies do you enjoy watching?" + }, + { + "speaker": "Tim", + "img_url": [ + "https://i.redd.it/h18us9b4sjfz.jpg" + ], + "blip_caption": "a photo of a dvd cover of a santa clause movie", + "query": "the santa clause dvd cover santa claus holiday", + "dia_id": "D8:28", + "text": "We love \"The Santa Clause\" too- it's so heartwarming and gets us all feeling festive!" + }, + { + "speaker": "John", + "img_url": [ + "https://i.pinimg.com/originals/d1/87/78/d187788dbb64158d2cbd6a1eaaa7e86b.jpg" + ], + "blip_caption": "a photo of a christmas tree with a lot of lights on it", + "query": "christmas tree colorful lights ornaments", + "dia_id": "D8:29", + "text": "\"The Santa Clause\" is a classic! It's so sweet and really captures the Christmas magic. It's just one of those movies that gets us all feeling festive. This was our tree last year." + }, + { + "speaker": "Tim", + "img_url": [ + "https://i.pinimg.com/originals/2d/be/40/2dbe40682781579e457e7ee5a0cc79fc.jpg" + ], + "blip_caption": "a photo of a christmas tree with a harry potter theme", + "query": "christmas tree harry potter ornament", + "dia_id": "D8:30", + "text": "Yep, it really does. That tree pic looks awesome! It must add so much holiday cheer to your house. This was ours." + }, + { + "speaker": "John", + "dia_id": "D8:31", + "text": "That looks awesome! Where did you get this tree?" + }, + { + "speaker": "Tim", + "blip_caption": "a photo of a christmas tree with a harry potter theme", + "dia_id": "D8:32", + "text": "I decorated this tree myself, going all out with a Harry Potter theme! It was a blast!" + }, + { + "speaker": "John", + "dia_id": "D8:33", + "text": "That themed tree looks amazing! You really know how to get the vibes just right!" + }, + { + "speaker": "Tim", + "dia_id": "D8:34", + "text": "Thanks! It was such a fun project and I'm really happy with how it turned out." + }, + { + "speaker": "John", + "dia_id": "D8:35", + "text": "Glad you had fun!" + }, + { + "speaker": "Tim", + "dia_id": "D8:36", + "text": "Great catching up! Take care, talk soon." + }, + { + "speaker": "John", + "dia_id": "D8:37", + "text": "Catch ya later! Talk soon. Take care and enjoy the rest of your day." + } + ], + "session_9_date_time": "6:59 pm on 26 August, 2023", + "session_9": [ + { + "speaker": "Tim", + "img_url": [ + "https://i.redd.it/nziqtijj65i11.jpg" + ], + "blip_caption": "a photo of a stack of books on a table", + "query": "stack of textbooks and notes", + "dia_id": "D9:1", + "text": "Hey John, this week's been really busy for me. Assignments and exams are overwhelming. I'm not giving up though! I'm trying to find a way to juggle studying with my fantasy reading hobby. How have you been?" + }, + { + "speaker": "John", + "img_url": [ + "https://i.pinimg.com/originals/79/c5/6c/79c56ca8e8173619eb96c1058eb5bb67.jpg" + ], + "blip_caption": "a photo of a group of girls basketball players posing for a picture", + "query": "group friends basketball posing", + "dia_id": "D9:2", + "text": "Hey Tim! I know the stress of exams and homework, but you got this! I'm doing OK, cheers for asking. Last week I visited home and caught up with my family and old friends. We had a great time talking about our childhood - it reminds me of the good ol' times!" + }, + { + "speaker": "Tim", + "dia_id": "D9:3", + "text": "Thanks for the pic! That group looks like a great squad. How long did you all play together?" + }, + { + "speaker": "John", + "dia_id": "D9:4", + "text": "We were teammates for four years in high school, so we've played together for quite some time. Have you ever been part of a sports team?" + }, + { + "speaker": "Tim", + "img_url": [ + "https://i.redd.it/bq2ogzcjuzz41.jpg" + ], + "blip_caption": "a photo of a book shelf with books and a clock", + "query": "harry potter books shelf", + "dia_id": "D9:5", + "text": "Nope, never been on a sports team. I'm more into reading and fantasy novels. I love sinking into different magical worlds. It's one of the reasons I love traveling to new places, to experience a different kind of magic." + }, + { + "speaker": "John", + "img_url": [ + "https://i.pinimg.com/originals/90/49/55/904955fe77567cf689d7db0ce606717d.jpg" + ], + "blip_caption": "a photo of a cityscape with a view of a skyscraper", + "query": "new york city skyline", + "dia_id": "D9:6", + "text": "Wow, Tim, that's an awesome book collection! It's cool to escape to different worlds with a hobby. By the way, I love discovering new cities - check out this pic from one of my trips to New York City!" + }, + { + "speaker": "Tim", + "dia_id": "D9:7", + "text": "Wow! That skyline looks amazing - I've been wanting to visit NYC. How was it?" + }, + { + "speaker": "John", + "dia_id": "D9:8", + "text": "Thanks! It was amazing. Everywhere you go there's something new and exciting. Exploring the city and trying all the restaurants was awesome. It's a must-visit!" + }, + { + "speaker": "Tim", + "dia_id": "D9:9", + "text": "Adding NYC to my travel list, sounds like a great adventure! I heard there's so much to explore and try out. Can't wait to visit!" + }, + { + "speaker": "John", + "dia_id": "D9:10", + "text": "Trust me, NYC is amazing! It's got so much to check out - the culture, food - you won't regret it. It's an adventure you'll never forget!" + }, + { + "speaker": "Tim", + "dia_id": "D9:11", + "text": "Woohoo! Sounds like a fun place with lots of potential. Can't wait to experience it for myself!" + }, + { + "speaker": "John", + "dia_id": "D9:12", + "text": "Awesome! Can't wait to hear when you are going. Let me know and I'm sure I can help you out." + }, + { + "speaker": "Tim", + "dia_id": "D9:13", + "text": "Yep, I'll let you know! Thanks for being so helpful." + }, + { + "speaker": "John", + "dia_id": "D9:14", + "text": "Sure thing! Any time you need help, don't hesitate to reach out." + }, + { + "speaker": "Tim", + "dia_id": "D9:15", + "text": "Thanks! Your support means a lot to me. Bye!" + } + ], + "session_10_date_time": "2:52 pm on 31 August, 2023", + "session_10": [ + { + "speaker": "Tim", + "dia_id": "D10:1", + "text": "Hey John, it's been a few days! I got a no for a summer job I wanted which wasn't great but I'm staying positive. On your NYC trip, did you have any troubles? How did you handle them?" + }, + { + "speaker": "John", + "dia_id": "D10:2", + "text": "Hey Tim! Sorry to hear about the job, but your positivity will help you find something great! My trip went okay - I had some trouble figuring out the subway at first, but then it was easy after someone helped explain it. How about you? Anything new you've tackled?" + }, + { + "speaker": "Tim", + "dia_id": "D10:3", + "text": "Thanks! Appreciate your encouragement. Yesterday, I tackled something new - I gave a presentation in class. I was nervous but I made it. Small step, but feels like progress." + }, + { + "speaker": "John", + "img_url": [ + "https://i.redd.it/zr17msb27uy91.jpg" + ], + "blip_caption": "a photo of a bowl of soup with a spoon and a butternut on a cutting board", + "query": "beautifully plated chicken soup", + "dia_id": "D10:4", + "text": "Cool, Tim! Taking the plunge and presenting can be tough, but awesome work! Progress is progress, keep it up. By the way, I've been trying out cooking recipes. Made this tasty soup recently - it was real good!" + }, + { + "speaker": "Tim", + "dia_id": "D10:5", + "text": "Wow, that looks great! How did you make it? Do you have a recipe you can share?" + }, + { + "speaker": "John", + "dia_id": "D10:6", + "text": "Thanks, I just sort of made it up on the spot so I don't have a recipe." + }, + { + "speaker": "Tim", + "dia_id": "D10:7", + "text": "That's ok! I can look some up. Can you tell me what spices you used in the soup?" + }, + { + "speaker": "John", + "dia_id": "D10:8", + "text": "I added some sage for a nice flavor. Enjoy!" + }, + { + "speaker": "Tim", + "dia_id": "D10:9", + "text": "Thanks! Excited to try this. Love experimenting with spices. By the way, have you been to Universal Studios? Planning a trip there next month." + }, + { + "speaker": "John", + "dia_id": "D10:10", + "text": "Cool! Haven't been there yet, but I've heard great things about Universal Studios. It's definitely on my bucket list. Have you been before?" + }, + { + "speaker": "Tim", + "dia_id": "D10:11", + "text": "Nope, but it's my first time going. I'm super stoked for the Harry Potter stuff. Can't wait!" + }, + { + "speaker": "John", + "dia_id": "D10:12", + "text": "Cool! It's gonna be a blast, like stepping into another world. Have a great time!" + }, + { + "speaker": "Tim", + "dia_id": "D10:13", + "text": "Thanks! I'll definitely have a blast. I'll let you know how it goes!" + }, + { + "speaker": "John", + "dia_id": "D10:14", + "text": "Great! Can't wait to hear about it. Have a safe trip!" + }, + { + "speaker": "Tim", + "dia_id": "D10:15", + "text": "Thanks! I'll make sure to have a safe trip." + }, + { + "speaker": "John", + "dia_id": "D10:16", + "text": "Bye! Take care and let's catch up soon!" + }, + { + "speaker": "Tim", + "dia_id": "D10:17", + "text": "Take care! Can't wait to catch up. Talk soon!" + } + ], + "session_11_date_time": "8:17 pm on 21 September, 2023", + "session_11": [ + { + "speaker": "John", + "dia_id": "D11:1", + "text": "Hey Tim, been a while! How ya been?" + }, + { + "speaker": "Tim", + "dia_id": "D11:2", + "text": "Hey John! Great to hear from you. Been busy with things, how about you?" + }, + { + "speaker": "John", + "dia_id": "D11:3", + "text": "Yeah, something cool happened! I attended a local restaurant with some new teammates last week. It was great getting to know them better." + }, + { + "speaker": "Tim", + "dia_id": "D11:4", + "text": "Good support is essential. How do you feel about them?" + }, + { + "speaker": "John", + "dia_id": "D11:5", + "text": "They're great friends. We connected over our shared love for basketball and had a ton of fun." + }, + { + "speaker": "Tim", + "dia_id": "D11:6", + "text": "Sounds awesome. Having friends who share your hobbies can be really fun. Any exciting plans with them?" + }, + { + "speaker": "John", + "dia_id": "D11:7", + "text": "We're planning to take a team trip next month to explore a new city and have some fun. Can't wait!" + }, + { + "speaker": "Tim", + "dia_id": "D11:8", + "text": "That sounds great! Exploring new cities is always so much fun. Where are you headed?" + }, + { + "speaker": "John", + "dia_id": "D11:9", + "text": "We're still deciding on the destination. Do you have any suggestions?" + }, + { + "speaker": "Tim", + "img_url": [ + "https://www.thebearandthefox.com/wp-content/uploads/2018/02/IMG_3494.jpg" + ], + "blip_caption": "a photo of a city with a clock tower and a sun setting", + "query": "edinburgh scotland castle sunset", + "dia_id": "D11:10", + "text": "Edinburgh, Scotland would be great for a magical vibe. It's the birthplace of Harry Potter and has awesome history and architecture. Plus, it's a beautiful city. What do you think?" + }, + { + "speaker": "John", + "dia_id": "D11:11", + "text": "That sounds like a great idea! I haven't been to Edinburgh yet, but it definitely sounds like a place worth considering for our trip. Thanks for the suggestion!" + }, + { + "speaker": "Tim", + "dia_id": "D11:12", + "text": "Glad you liked it. Let me know if you need any more suggestions." + }, + { + "speaker": "John", + "img_url": [ + "https://c8.alamy.com/zooms/9/de5f1d4e73244a8f94b16a6b6d093748/ttnmea.jpg" + ], + "blip_caption": "a photo of a basketball ball on the ground with a basketball hoop in the background", + "query": "basketball court sunset", + "dia_id": "D11:13", + "text": "Thanks! I'll definitely reach out if I need more suggestions. Appreciate the help! Here's a pic I snapped during one of our practices. The sunset looked amazing on the court. Moments like these make me so grateful for my basketball career." + }, + { + "speaker": "Tim", + "dia_id": "D11:14", + "text": "Wow, that looks amazing! What do you love most about your basketball career?" + }, + { + "speaker": "John", + "dia_id": "D11:15", + "text": "Thanks! I love playing pro ball - it's a constant challenge and keeps me growing. There's nothing like seeing myself get better and beating goals. Plus, playing with my teammates and having the fans cheer is awesome. Basketball gives me a great sense of satisfaction and purpose." + }, + { + "speaker": "Tim", + "dia_id": "D11:16", + "text": "It's great that you have a passion that helps you grow and reach your goals. Achieving and feeling fulfilled must be amazing. Do you have any specific targets or goals you're working towards?" + }, + { + "speaker": "John", + "dia_id": "D11:17", + "text": "Definitely! I'm focusing on better shooting and making more of an impact on the court. I want to be known as a consistent performer and help my team. Off the court, I'm also looking into more endorsements and building my brand. It's important for me to think about life after basketball." + }, + { + "speaker": "Tim", + "dia_id": "D11:18", + "text": "Awesome! It's great that you have goals both on and off the court. It's wise to think about the future and building your brand. What are your thoughts on life after basketball?" + }, + { + "speaker": "John", + "dia_id": "D11:19", + "text": "I've thought about it a lot. I want to use my platform to make a positive difference and inspire others - maybe even start a foundation and do charity work. It's important to me to make the most of the chances I get and leave a meaningful legacy." + }, + { + "speaker": "Tim", + "dia_id": "D11:20", + "text": "Wow, that's amazing. Good on you for wanting to make a difference and motivate others. I'm sure you'll succeed! Is there anything I can do to support you?" + }, + { + "speaker": "John", + "dia_id": "D11:21", + "text": "Thanks! I'm trying to figure out how to pick the right ones - any advice on that?" + }, + { + "speaker": "Tim", + "blip_caption": "a photo of a stack of books sitting on top of a counter", + "dia_id": "D11:22", + "text": "When picking endorsements, make sure they align with your values and brand. Look for a company that shares your desire to make a change and help others. It's important that the endorsement feels authentic to your followers." + }, + { + "speaker": "John", + "dia_id": "D11:23", + "text": "Sounds like good advice! I was wondering if you have any book recommendations for my trip?" + }, + { + "speaker": "Tim", + "img_url": [ + "https://live.staticflickr.com/3195/3322703152_057a33dca1_z.jpg" + ], + "blip_caption": "a photography of a book cover with a man in a hooded jacket", + "query": "the name of the wind book cover fantasy cloaked figure burning city", + "dia_id": "D11:24", + "re-download": true, + "text": "Yeah! I think you'd love this fantasy novel by Patrick Rothfuss. It's a book that'll take you to a different world. Great for you when you're traveling. Have fun!" + }, + { + "speaker": "John", + "img_url": [ + "https://i0.wp.com/www.themself.org/wp-content/uploads/2016/10/img_20161018_133948.jpg" + ], + "blip_caption": "a photo of a bookshelf with a lot of books on it", + "query": "bookshelf", + "dia_id": "D11:25", + "text": "Thanks! I'll definitely check it out - looks like a great book to read while traveling. Can't wait to dive into it! Here's a photo of my bookshelf. You can see some of the books I've read and enjoyed." + }, + { + "speaker": "Tim", + "dia_id": "D11:26", + "text": "Great bookshelf! I saw that you had \"The Alchemist\" on there, one of my favorites. Did you enjoy it?" + }, + { + "speaker": "John", + "dia_id": "D11:27", + "text": "Yep, I read that book and loved it! It made me think about life and how important it is to follow one's dreams. Highly recommend it!" + }, + { + "speaker": "Tim", + "dia_id": "D11:28", + "text": "Glad you liked it! \"The Alchemist\" is worth it." + }, + { + "speaker": "John", + "dia_id": "D11:29", + "text": "Thanks! Take care!" + }, + { + "speaker": "Tim", + "dia_id": "D11:30", + "text": "Have fun! Take care and talk to you soon." + } + ], + "session_12_date_time": "3:00 pm on 2 October, 2023", + "session_12": [ + { + "speaker": "Tim", + "blip_caption": "a photo of a bookcase filled with dvds and games", + "dia_id": "D12:1", + "text": "Hey John! Awesome catchin' up with you! A lot's changed since last time." + }, + { + "speaker": "John", + "img_url": [ + "https://platinumnotary.files.wordpress.com/2023/03/img-6679.jpg" + ], + "blip_caption": "a photo of a wedding ceremony in a greenhouse with people taking pictures", + "query": "wedding ceremony", + "dia_id": "D12:2", + "text": "Hey, Tim! Good to hear from you. Anyway, a lot has been going on with me. My girlfriend and I had an amazing and emotional wedding ceremony last week." + }, + { + "speaker": "Tim", + "dia_id": "D12:3", + "text": "Congrats! That was such a special day! How did you manage to have the ceremony during these times?" + }, + { + "speaker": "John", + "dia_id": "D12:4", + "text": "Thanks! We were lucky to find a lovely greenhouse venue for a smaller, more intimate gathering. We made sure to follow the necessary safety protocols and ensure that everyone felt safe. It was wonderful to have our loved ones celebrate with us." + }, + { + "speaker": "Tim", + "dia_id": "D12:5", + "text": "Awesome! It sounds like you found a great venue and had your loved ones celebrate with you. Weddings are definitely full of joy!" + }, + { + "speaker": "John", + "dia_id": "D12:6", + "text": "Yeah! Such a great day! It was so beautiful having everyone celebrating with us. I'd never felt so much love and happiness before. Some of my hiking club friends came even though I just joined!" + }, + { + "speaker": "Tim", + "dia_id": "D12:7", + "text": "Wow! Your wedding day must've been so special. Love sure does bring us joy, huh? Any favorite memories from the big day?" + }, + { + "speaker": "John", + "img_url": [ + "https://images.pexels.com/photos/8815283/pexels-photo-8815283.jpeg" + ], + "blip_caption": "a photography of a bride walking down the aisle with her groom", + "query": "wife walking down the aisle moment", + "dia_id": "D12:8", + "re-download": true, + "text": "Oh yeah! Picking a favorite memory was tough, but seeing her walking down the aisle, her face lit up and it got me all choked up. What a magical moment - one I'll always remember." + }, + { + "speaker": "Tim", + "blip_caption": "a photo of a desk with a bookcase full of books and a lot of other items", + "dia_id": "D12:9", + "text": "I bet that moment was unreal! One moment can make such an impression. I saw your photo of you two dancing, it must have been incredible. Wanna tell me more?" + }, + { + "speaker": "John", + "dia_id": "D12:10", + "text": "That dance was great! We had our first dance at a cozy restaurant. It was so dreamy with the music and candlelight. We were so lucky to have everyone with us!" + }, + { + "speaker": "Tim", + "dia_id": "D12:11", + "text": "Wow, a candlelit dance in a cozy restaurant! Sounds like a fairytale! So special to share with your loved ones. Weddings truly are the best!" + }, + { + "speaker": "John", + "dia_id": "D12:12", + "text": "Thanks! It was a great day. Having everyone there made it extra special. It's moments like these that bring love and joy." + }, + { + "speaker": "Tim", + "dia_id": "D12:13", + "text": "Congrats again! Love is truly magical and brings so much joy. I'm so happy for you and your new wife!" + }, + { + "speaker": "John", + "dia_id": "D12:14", + "text": "Thanks so much! Your words mean a lot. I'm lucky to have you in my life, bringing so much love and joy." + }, + { + "speaker": "Tim", + "img_url": [ + "https://i.redd.it/z6jusjhd0pd71.jpg" + ], + "blip_caption": "a photo of a bookcase filled with dvds and games", + "query": "bookshelf fantasy novels", + "dia_id": "D12:15", + "text": "Thanks. Your friendship means a lot to me. I'm here for you anytime. I also wanted to share this bookshelf with you. It's filled with my favorite fantasy novels." + }, + { + "speaker": "John", + "dia_id": "D12:16", + "text": "Cool! What do you enjoy about them so much?" + }, + { + "speaker": "Tim", + "dia_id": "D12:17", + "text": "They really fire up my imagination and take me to alternate realities. They're my escape from reality." + }, + { + "speaker": "John", + "img_url": [ + "https://i.redd.it/odlwr40mol581.jpg" + ], + "blip_caption": "a photo of a bunch of basketball jerseys laying on a bed", + "query": "basketball jerseys collection", + "dia_id": "D12:18", + "text": "That's great Tim! Books and movies make us escape to different places. I like to collect jerseys." + }, + { + "speaker": "Tim", + "dia_id": "D12:19", + "text": "Cool! Who's your favorite basketball team/player?" + }, + { + "speaker": "John", + "dia_id": "D12:20", + "text": "Thanks! The Wolves are my team for sure. And LeBron is the man - love his skills and leadership." + }, + { + "speaker": "Tim", + "dia_id": "D12:21", + "text": "The Wolves are solid and LeBron's skills and leadership are amazing." + }, + { + "speaker": "John", + "blip_caption": "a photo of a group of people standing on a basketball court", + "dia_id": "D12:22", + "text": "I really admire his work ethic and dedication to the game, such an inspiration." + }, + { + "speaker": "Tim", + "dia_id": "D12:23", + "text": "LeBron is incredible. Have you ever had the opportunity to meet him or see him play live?" + }, + { + "speaker": "John", + "dia_id": "D12:24", + "text": "Yeah, I've had the chance to meet him a few times. He's real chill and it was a wild experience. I also had the opportunity to see him play live. It was tough, but I learned a lot from watching a top player like him." + }, + { + "speaker": "Tim", + "blip_caption": "a photo of a crowd of people at a concert with their hands in the air", + "dia_id": "D12:25", + "text": "Wow! Meeting him and seeing him play must've been awesome. I can only imagine his skills up close." + }, + { + "speaker": "John", + "img_url": [ + "https://get.pxhere.com/photo/structure-crowd-audience-basketball-stadium-arena-sports-miami-sport-venue-soccer-specific-stadium-music-venue-basketball-moves-slam-dunk-miami-heat-game-miami-heat-1383801.jpg" + ], + "blip_caption": "a photo of a basketball game with a crowd of people watching", + "query": "basketball game crowd", + "dia_id": "D12:26", + "text": "It was so amazing! The energy was crazy and everyone was so hyped. Seeing the top players was so motivating - it made me want to work even harder. Check out this pic from the game!" + }, + { + "speaker": "Tim", + "dia_id": "D12:27", + "text": "That crowd looks wild! The game must've been amazing!" + }, + { + "speaker": "John", + "dia_id": "D12:28", + "text": "The energy was incredible! It's moments like these that make me love my job." + }, + { + "speaker": "Tim", + "dia_id": "D12:29", + "text": "Cool! It's great to find something you enjoy doing. Keep going for it! See ya later!" + } + ], + "session_13_date_time": "1:50 pm on 13 October, 2023", + "session_13": [ + { + "speaker": "Tim", + "dia_id": "D13:1", + "text": "Hey John! It's been ages since we last talked. Guess what? Last week I went to a Harry Potter conference in the UK - it was incredible! There were so many people who shared the same love of HP as me, it was like a magical family. I felt so inspired and like I got a new lease of life. I love how my passion for fantasy stuff brings me closer to people from all over the world, it's pretty cool." + }, + { + "speaker": "John", + "img_url": [ + "https://upload.wikimedia.org/wikipedia/commons/4/4c/USA_men%27s_national_basketball_team_%2851910110377%29.jpg" + ], + "blip_caption": "a photography of a basketball team posing for a team photo", + "query": "group photo basketball", + "dia_id": "D13:2", + "re-download": true, + "text": "Hey Tim! Great to hear from you. It's awesome how our passions connect us with others, yeah? You sound like you fit right in and got a real buzz out of it. I feel the same way with my team." + }, + { + "speaker": "Tim", + "dia_id": "D13:3", + "text": "Wow, you guys look great! How have games been going?" + }, + { + "speaker": "John", + "dia_id": "D13:4", + "text": "It was an intense season with both tough losses and great wins. Overall, I'd say we did pretty well." + }, + { + "speaker": "Tim", + "dia_id": "D13:5", + "text": "Cool! Sounds like you guys had some tough games. How did you handle those?" + }, + { + "speaker": "John", + "img_url": [ + "https://tapinto-production.s3.amazonaws.com/uploads/articles/im/best_9e2201886c43d264dbef_IMG_6460.jpg" + ], + "blip_caption": "a photo of a soccer team posing for a picture with a trophy", + "query": "teammates celebration victory", + "dia_id": "D13:6", + "text": "Thanks! We faced tough opponents but that's what drives us to get better. We back each other up and won't quit." + }, + { + "speaker": "Tim", + "dia_id": "D13:7", + "text": "Congrats! That's awesome. It must feel good, right?" + }, + { + "speaker": "John", + "img_url": [ + "https://i.redd.it/wheq5bijuqc91.jpg" + ], + "blip_caption": "a photo of a man holding a trophy in front of a crowd", + "query": "championship trophy holding up", + "dia_id": "D13:8", + "text": "Yeah, it feels great! All that hard work and effort was totally worth it. We even won a trophy!" + }, + { + "speaker": "Tim", + "dia_id": "D13:9", + "text": "Way to go! You must have been elated up there with that trophy. All the hard work paid off! Congrats - I'm so proud of you. Keep it up!" + }, + { + "speaker": "John", + "dia_id": "D13:10", + "text": "Thanks! I was definitely elated. Your support really means a lot to me. I'll keep working hard." + }, + { + "speaker": "Tim", + "dia_id": "D13:11", + "text": "No problem! I'm here for you anytime. Keep believing in yourself!" + }, + { + "speaker": "John", + "dia_id": "D13:12", + "text": "Thanks! Appreciate your support. Always staying filled with self-belief." + }, + { + "speaker": "Tim", + "blip_caption": "a photo of a box of serenityy memory foam", + "dia_id": "D13:13", + "text": "You got this! Stay motivated and remember that anything is possible with hard work. Keep pushing for your goals!" + }, + { + "speaker": "John", + "dia_id": "D13:14", + "text": "Thanks! Your encouragement means a lot to me. I'm feeling motivated and ready to keep pushing for my goals! I'm going to need some new shoes after all these games though." + }, + { + "speaker": "Tim", + "img_url": [ + "http://www.binxberryconsignment.com/cdn/shop/products/IMG_3958.jpg" + ], + "blip_caption": "a photo of a pair of black and pink running shoes", + "query": "memory foam shoes", + "dia_id": "D13:15", + "text": "Glad my encouragement helped! These are amazing - like walking on clouds! Game changer!" + }, + { + "speaker": "John", + "dia_id": "D13:16", + "text": "They look comfortable. Where did you get them?" + }, + { + "speaker": "Tim", + "dia_id": "D13:17", + "text": "I got them online - they're super comfy! Definitely recommend!" + }, + { + "speaker": "John", + "dia_id": "D13:18", + "text": "Cheers! I'll definitely check them out. Thanks for the recommendation!" + }, + { + "speaker": "Tim", + "dia_id": "D13:19", + "text": "No worries. Let me know if there's anything else I can assist you with. Always here to help!" + }, + { + "speaker": "John", + "dia_id": "D13:20", + "text": "Thanks! Appreciate it. I'll reach out if I need anything." + }, + { + "speaker": "Tim", + "dia_id": "D13:21", + "text": "Cool! Stay motivated and keep chasing those dreams! Chat soon!" + }, + { + "speaker": "John", + "dia_id": "D13:22", + "text": "Thanks! I'll definitely stay motivated and keep chasing those dreams. You too, keep up the passion. Talk soon!" + } + ], + "session_14_date_time": "1:50 pm on 17 October, 2023", + "session_14": [ + { + "speaker": "John", + "dia_id": "D14:1", + "text": "Hey Tim! Long time no talk - a lot has been going on since then!" + }, + { + "speaker": "Tim", + "dia_id": "D14:2", + "text": "Hey John! Long time no see! Can't wait to catch up and hear all about what you've been up to." + }, + { + "speaker": "John", + "dia_id": "D14:3", + "text": "Seems like forever since we caught up! I'm now mentoring the younger players on my team. It's super rewarding and I'm loving sharing my skills and knowledge with them. It's also a great way for me to stay involved in the game during the off-season." + }, + { + "speaker": "Tim", + "dia_id": "D14:4", + "text": "Wow! Mentoring must be so rewarding. You get to show others what you know - that's awesome! Is it difficult? Any hiccups?" + }, + { + "speaker": "John", + "dia_id": "D14:5", + "text": "There are challenges, since everyone is so different. But it's been awesome gaining experience and adapting, motivating and encouraging everyone. It's been great to watch each of them develop and reach their goals - such a reward!" + }, + { + "speaker": "Tim", + "dia_id": "D14:6", + "text": "Wow, that's awesome! It must be really rewarding to see them reach their goals. What's it like mentoring them?" + }, + { + "speaker": "John", + "img_url": [ + "https://cdn2.picryl.com/photo/2020/01/16/members-of-the-local-and-us-communities-attend-the-edfa79-1024.jpg" + ], + "blip_caption": "a photography of a basketball player standing in a gym with his hands on his hips", + "query": "mentoring young players practice smiles", + "dia_id": "D14:7", + "re-download": true, + "text": "Mentoring them has been awesome! Seeing their growth, improvement, and confidence is so fulfilling. I'm glad I could make a positive impact on their lives. Here's a pic of me and some of the younger players at a recent practice." + }, + { + "speaker": "Tim", + "dia_id": "D14:8", + "text": "You're really doing great with them. Do any of them see you as a mentor?" + }, + { + "speaker": "John", + "dia_id": "D14:9", + "text": "Some of them do see me as a mentor, which is really rewarding. I try to provide them with advice and support on and off the court. Being a positive role model for them is something I enjoy." + }, + { + "speaker": "Tim", + "dia_id": "D14:10", + "text": "That's incredible! How does it feel to have their trust and admiration? It must be such an honor to be a positive role model for them." + }, + { + "speaker": "John", + "dia_id": "D14:11", + "text": "It feels great to have their trust and admiration. Being a role model for these young athletes is so fulfilling. I'm glad my experiences can help shape their future and inspire them to go after their dreams." + }, + { + "speaker": "Tim", + "img_url": [ + "https://images.pexels.com/photos/12312263/pexels-photo-12312263.jpeg?cs\\u003dsrgb\\u0026dl\\u003dpexels-tanya-badillo-12312263.jpg" + ], + "blip_caption": "a photo of a sunset over a mountain range with a few trees", + "query": "mountain range sunset", + "dia_id": "D14:12", + "text": "You're doing a great job with them. Way to go! This is what I've been up to." + }, + { + "speaker": "John", + "dia_id": "D14:13", + "text": "Wow, stunning! And thanks. Really appreciate it. Means a lot." + }, + { + "speaker": "Tim", + "img_url": [ + "https://i.redd.it/hf54pvtf8rt51.jpg" + ], + "blip_caption": "a photo of a sunset over a mountain with a tree", + "query": "sunset blue ridge mountains north carolina", + "dia_id": "D14:14", + "text": "I took this pic last summer. Seeing it was so stunning. Thanks for appreciating it. It means a lot to me." + }, + { + "speaker": "John", + "dia_id": "D14:15", + "text": "Where did you capture this? Nature is truly amazing, isn't it?" + }, + { + "speaker": "Tim", + "dia_id": "D14:16", + "text": "I snapped that pic on my trip to the Smoky Mountains last year. It was incredible seeing it in person. Nature's really something else!" + }, + { + "speaker": "John", + "dia_id": "D14:17", + "text": "Yeah, it's amazing how nature's beauty and grandeur can take our breath away. It's so nice to escape the noise of the city and relax in nature. Good for you to get to enjoy that stunning view!" + }, + { + "speaker": "Tim", + "dia_id": "D14:18", + "text": "Nature is indeed refreshing. A good break from school." + }, + { + "speaker": "John", + "dia_id": "D14:19", + "text": "How are you doing in shcool?" + }, + { + "speaker": "Tim", + "dia_id": "D14:20", + "text": "Doing good! Busy with studies but finding time to relax with books - good balance." + }, + { + "speaker": "John", + "dia_id": "D14:21", + "text": "Cool! Finding that balance is key. Are you currently reading any books?" + }, + { + "speaker": "Tim", + "dia_id": "D14:22", + "text": "I'm reading this book and I'm totally hooked! What about you?" + }, + { + "speaker": "John", + "dia_id": "D14:23", + "text": "I haven't had much time to read, but after we talked I finally picked up a book and it's been awesome! Talk to you later!" + } + ], + "session_15_date_time": "5:51 pm on 21 October, 2023", + "session_15": [ + { + "speaker": "Tim", + "dia_id": "D15:1", + "text": "Hey John! Haven't talked to you in a bit but wanted to let you know I read this awesome book about castles in the UK. It was so interesting and blew me away! I dream of visiting them one day." + }, + { + "speaker": "John", + "img_url": [ + "https://creatingmewp.files.wordpress.com/2023/05/img_9047.jpg" + ], + "blip_caption": "a photo of a man sitting on a bench overlooking a cliff", + "query": "castle scotland", + "dia_id": "D15:2", + "text": "Hey Tim! Great to hear from you. Learning about different cultures and seeing historical architecture fascinates me. Visiting castles is really on my bucket list. Just look at this one; what a sight! I'm so excited to explore the world and experience these gorgeous places. On that note, how's your fantasy writing going?" + }, + { + "speaker": "Tim", + "dia_id": "D15:3", + "text": "That castle looks amazing! I hope I get to visit it someday. My writing is going well: I'm in the middle of fantasy novel and it's a bit nerve-wracking but so exciting! All my hard work is paying off. Writing brings such joy and it's incredible how it can create a whole new world. Thanks so much for believing in me!" + }, + { + "speaker": "John", + "dia_id": "D15:4", + "text": "That's great! I'm glad your writing is going well. It must be exciting to see it all come together. Keep going! Do you have a specific source of inspiration for your stories?" + }, + { + "speaker": "Tim", + "dia_id": "D15:5", + "text": "Thanks! Books, movies, and real-life experiences all fire up my creativity. For example, reading about castles in the UK gave me loads of ideas. Plus, certain authors are like goldmines of inspiration for me. Connecting with the things I love makes writing even more fun." + }, + { + "speaker": "John", + "dia_id": "D15:6", + "text": "Wow! Sounds like a great mix. Is there a particular author whose work inspires you?" + }, + { + "speaker": "Tim", + "img_url": [ + "https://i.redd.it/4mff2zeq18381.jpg" + ], + "blip_caption": "a photo of a book with a page in it on a table", + "query": "harry potter sorcerer's stone well-read copy", + "dia_id": "D15:7", + "text": "J.K. Rowling is such an inspiring writer. Her books are so captivating with their detail and creative storytelling. She can definitely transport readers into another world and make them feel so much. I'm always taking notes on her style for my own writing." + }, + { + "speaker": "John", + "dia_id": "D15:8", + "text": "Cool! How long have you been reading her works?" + }, + { + "speaker": "Tim", + "dia_id": "D15:9", + "text": "I've been reading her stuff for a long time. Her stories have been with me and still inspire me. There's something special about her writing that really speaks to me." + }, + { + "speaker": "John", + "dia_id": "D15:10", + "text": "Wow, some authors really have such an influence on us! They become part of our life and affect our interests. Do you have a favorite J.K. Rowling quote?" + }, + { + "speaker": "Tim", + "dia_id": "D15:11", + "text": "Yeah! There's a quote by J.K. Rowling that I really like: \"Turn on the light - happiness hides in the darkest of times.\" That's how I keep hope alive during tough times." + }, + { + "speaker": "John", + "img_url": [ + "https://i.pinimg.com/originals/5e/f4/f9/5ef4f9bd2f094b2d0a4ddd4861b928a0.jpg" + ], + "blip_caption": "a photo of a white board with a drawing of arrows and words", + "query": "inspirational quote whiteboard", + "dia_id": "D15:12", + "text": "Nice quote! It reminds us to stay positive and find joy even in hard times. It's a guiding light when things get rough. I appreciate you sharing it!" + }, + { + "speaker": "Tim", + "dia_id": "D15:13", + "text": "Nice job, John! What did you write on that whiteboard?" + }, + { + "speaker": "John", + "dia_id": "D15:14", + "text": "On that whiteboard, I wrote down some motivational quotes and strategies to help me stay focused and push through tough workouts. It really helps me stay motivated and keep improving." + }, + { + "speaker": "Tim", + "blip_caption": "a photo of a woman holding a plaque in front of a wall", + "dia_id": "D15:15", + "text": "That's awesome! Visual reminders and strategies can really help in staying motivated. It's cool that you have those quotes to keep you going during tough workouts." + }, + { + "speaker": "John", + "img_url": [ + "https://images.rawpixel.com/image_social_landscape/czNmcy1wcml2YXRlL3Jhd3BpeGVsX2ltYWdlcy93ZWJzaXRlX2NvbnRlbnQvbHIvbnMxODI2MC1pbWFnZS1rd3Z3bmQxZi5qcGc.jpg" + ], + "blip_caption": "a photography of a desk with a laptop and a lightbox that says make it happen", + "query": "plaque believe power within motivation", + "dia_id": "D15:16", + "re-download": true, + "text": "This plaque I keep on my desk is a constant reminder to believe in myself. It helps me trust my abilities and face any obstacles. Having physical reminders like this really helps me stay motivated." + }, + { + "speaker": "Tim", + "dia_id": "D15:17", + "text": "That's awesome! What keeps you motivated during challenging times?" + }, + { + "speaker": "John", + "img_url": [ + "https://www.americustimesrecorder.com/wp-content/uploads/sites/43/2023/04/team-huddle-rotated.jpg" + ], + "blip_caption": "a photo of a group of women soccer players huddle together", + "query": "team huddle game", + "dia_id": "D15:18", + "text": "My teammates believing in me and my love for improving my skills keep me going, even when things get tough. I don't want to let them down." + }, + { + "speaker": "Tim", + "dia_id": "D15:19", + "text": "Nice one! What do you reckon makes them such a good support?" + }, + { + "speaker": "John", + "dia_id": "D15:20", + "text": "They always support me, even when I make mistakes. Their encouragement keeps me going." + }, + { + "speaker": "Tim", + "blip_caption": "a photo of a group of people hiking through a forest", + "dia_id": "D15:21", + "text": "That's key, having a strong support network can really help with what we're trying to do. Do you have people you can lean on outside of sports?" + }, + { + "speaker": "John", + "dia_id": "D15:22", + "text": "Yeah, I'm lucky - I have people who are super supportive, always there for me no matter what." + }, + { + "speaker": "Tim", + "img_url": [ + "https://i.pinimg.com/originals/85/59/73/855973c53d20fad61ea048269f29fadb.jpg" + ], + "blip_caption": "a photo of a group of women sitting on the grass eating", + "query": "group friends beach picnic", + "dia_id": "D15:23", + "text": "Awesome! Having people who lift us up is essential. I'm grateful I have friends and family who support me - it's huge." + }, + { + "speaker": "John", + "img_url": [ + "https://images.pexels.com/photos/1655329/pexels-photo-1655329.jpeg?cs\\u003dsrgb\\u0026dl\\u003dpexels-lisa-fotios-1655329.jpg" + ], + "blip_caption": "a photo of a group of people standing around a kitchen table", + "query": "family dinner", + "dia_id": "D15:24", + "text": "Having loved ones who support us is so important. My family is always there for me." + }, + { + "speaker": "Tim", + "dia_id": "D15:25", + "text": "Wow, look at this great group! Are these your people?" + }, + { + "speaker": "John", + "img_url": [ + "https://s3-us-west-2.amazonaws.com/sportshub2-uploads-prod/files/sites/1567/2018/02/09230004/IMG_8348-e1518217261806.jpg" + ], + "blip_caption": "a photo of a group of people standing on a basketball court", + "query": "group of basketball players cheering on sidelines", + "dia_id": "D15:26", + "text": "Yeah, definitely! That's my fam hanging out. Being with them brings me so much happiness and helps me remember what's important. My team is like my second family too." + }, + { + "speaker": "Tim", + "blip_caption": "a photo of a group of kids playing a game of basketball", + "dia_id": "D15:27", + "text": "That looks fun! What else do you do with them?" + }, + { + "speaker": "John", + "dia_id": "D15:28", + "text": "What people usually do when you hang out with friends and family - movies, dinner out. And what are your favorite activities for fun?" + }, + { + "speaker": "Tim", + "img_url": [ + "https://images.pexels.com/photos/11818038/pexels-photo-11818038.jpeg?cs\\u003dsrgb\\u0026dl\\u003dpexels-ana-ps-11818038.jpg" + ], + "blip_caption": "a photo of a fire in a fireplace with a dog standing next to it", + "query": "campfire group people", + "dia_id": "D15:29", + "text": "I love going on road trips with friends and family, exploring and hiking or playing board games. And in my free time, I enjoy curling up with a good book, escaping reality and getting lost in different worlds. That's what I'm talking about." + }, + { + "speaker": "John", + "img_url": [ + "https://i.redd.it/6383i30xl5w31.jpg" + ], + "blip_caption": "a photo of a slow cooker filled with a mixture of food", + "query": "cooking meal", + "dia_id": "D15:30", + "text": "Yep, I totally get it! Cuddling up with a book is my chill time. And when I'm away from the court, cooking is therapy for me. It's a good way to be creative and experiment with flavors while taking a break. Here's a photo of me cooking a meal." + }, + { + "speaker": "Tim", + "dia_id": "D15:31", + "text": "That slow cooker meal looks yum! Cooking is a great way to chill and be creative. Do you have any favorite recipes you can show me?" + }, + { + "speaker": "John", + "blip_caption": "a photo of a pan of chicken and vegetables cooking on a stove", + "dia_id": "D15:32", + "text": "Definitely! I make this yummy honey garlic chicken with roasted veg a lot. It's one of my favorites. I'm always trying out new recipes, so let me know if you want suggestions!" + }, + { + "speaker": "Tim", + "dia_id": "D15:33", + "text": "Mmm, that sounds delicious, John! Can I get the recipe for it?" + }, + { + "speaker": "John", + "dia_id": "D15:34", + "text": "Sure thing! I can write it down for you and mail it to you." + }, + { + "speaker": "Tim", + "dia_id": "D15:35", + "text": "Can't wait to try it. Thanks for sharing the recipe!" + }, + { + "speaker": "John", + "dia_id": "D15:36", + "text": "No worries. Hope you enjoy it! Let me know how it turns out." + }, + { + "speaker": "Tim", + "dia_id": "D15:37", + "text": "Sure thing! Thanks. Great talking to you. Take care!" + }, + { + "speaker": "John", + "dia_id": "D15:38", + "text": "It was nice chatting with you. Talk to you later!" + } + ], + "session_16_date_time": "11:41 am on 6 November, 2023", + "session_16": [ + { + "speaker": "Tim", + "dia_id": "D16:1", + "text": "Hey John, long time no see! Hope you've been doing well. Since we last chat, some stuff's happened. Last week, I had a huge writing issue - got stuck on a plot twist and couldn't find my way out. It was crazy frustrating, but I kept pushing and eventually got the ideas flowing again." + }, + { + "speaker": "John", + "dia_id": "D16:2", + "text": "Hey Tim! Awesome to hear from you. Yeah, I get how that would've been so annoying! But you stuck it out, that's so cool. Same with me on the court. Just gotta find a way to tough it out and keep things flowing. Then when you make it through, it's all the more satisfying, right?" + }, + { + "speaker": "Tim", + "dia_id": "D16:3", + "text": "Yeah! It was hard but once it's over, the feeling is amazing. That's what makes it so beautiful, the struggle and then the satisfaction." + }, + { + "speaker": "John", + "dia_id": "D16:4", + "text": "Yeah! Struggles make it worth it. Like in sports, that's when the win feels great! Challenges force us to develop and become better." + }, + { + "speaker": "Tim", + "dia_id": "D16:5", + "text": "Overcoming challenges builds strength and pushes personal growth. It's about the journey and what we learn, not just winning. This was a great reminder for me. Got any examples from that sport you mentioned?" + }, + { + "speaker": "John", + "dia_id": "D16:6", + "text": "Yeah, last year I had a basketball game where we were trailing big time in the 4th quarter. We had to dig deep and keep on pushing to overturn the deficit and it was amazing when that final buzzer sounded. Unforgettable feeling." + }, + { + "speaker": "Tim", + "img_url": [ + "https://i.redd.it/w41ygbk6p4ea1.jpg" + ], + "blip_caption": "a photo of a basketball in a case with a signed ball", + "query": "basketball signed by favorite basketball player", + "dia_id": "D16:7", + "text": "Wow, that must have been an incredible feeling! You really showed determination and perseverance. Do you have any photos or keepsakes from the game? I'd love to see them! By the way, I have something special too - this is my prized possession, a basketball signed by my favorite player. It serves as a reminder of all the hard work." + }, + { + "speaker": "John", + "dia_id": "D16:8", + "text": "Wow! What makes your favorite player so inspiring? Do you have any special stories or moments with them?" + }, + { + "speaker": "Tim", + "dia_id": "D16:9", + "text": "I just love watching LeBron. There was this Finals game a few years back with an epic block that totally changed the game and ended up winning it. Seeing him go for it like that was such an inspiration - never give up, you know?" + }, + { + "speaker": "John", + "dia_id": "D16:10", + "text": "Remember that epic block in Game 7 of the '16 Finals? He chased down Iguodala and pinned the ball against the backboard. That kind of determination and heart is why I love basketball." + }, + { + "speaker": "Tim", + "dia_id": "D16:11", + "text": "Yeah, that's the one! It was awesome. Moments like that make me love sports and admire the players' determination and heart." + }, + { + "speaker": "John", + "dia_id": "D16:12", + "text": "LeBron's moments of determination and heart are incredible. It's why I enjoy playing and pushing myself. You never know when those special moments might occur, but it's always fun to be part of it." + }, + { + "speaker": "Tim", + "dia_id": "D16:13", + "text": "Those special moments make it all worth it. It's amazing to be part of something bigger and feel the joy and fulfillment. Keep pushing and having those moments on the court!" + }, + { + "speaker": "John", + "dia_id": "D16:14", + "text": "Speaking of special moments, my wife and I just left for our European vacation! It will be short but sweet. You've been before, any recommendations?" + }, + { + "speaker": "Tim", + "dia_id": "D16:15", + "text": "That's great! I hope you two have a great time. I would recommend visiting some castles, they are just so magical!" + }, + { + "speaker": "John", + "dia_id": "D16:16", + "text": "Thanks! We'll have to check some out. Wishing you all the best with everything you're pursuing. Stay safe!" + }, + { + "speaker": "Tim", + "dia_id": "D16:17", + "text": "Thanks! You too, buddy. Take it easy and keep going for it. Stay safe and let's stay in touch!" + } + ], + "session_17_date_time": "3:36 pm on 11 November, 2023", + "session_17": [ + { + "speaker": "John", + "dia_id": "D17:1", + "text": "Hey Tim! Great to chat again. So much has happened!" + }, + { + "speaker": "Tim", + "img_url": [ + "https://i.redd.it/7a8i8uea5au51.jpg" + ], + "blip_caption": "a photo of a book with a picture of a storm of swords", + "query": "worn copy game of thrones", + "dia_id": "D17:2", + "text": "Hey John! Great chatting with you as always. What's been happening lately? I've been reading as usual." + }, + { + "speaker": "John", + "img_url": [ + "https://familyadventuresva.files.wordpress.com/2022/03/img_5178.jpg" + ], + "blip_caption": "a photo of a group of people sitting on top of a mountain", + "query": "american west landscapes grand canyon", + "dia_id": "D17:3", + "text": "My wife and I were road tripping out on the European coastline, and it was amazing! The views were spectacular, and we had lots of fun bonding and creating amazing memories. It was such a nice change to my regular life." + }, + { + "speaker": "Tim", + "blip_caption": "a photo of a statue of a woman with a blue hat on", + "dia_id": "D17:4", + "text": "Wow! Sounds like an incredible road trip. I'm glad you and your wife had such a great time!" + }, + { + "speaker": "John", + "dia_id": "D17:5", + "text": "Thanks! Yeah, it was awesome. We got to see some epic spots. It's hard to describe how beautiful they were!" + }, + { + "speaker": "Tim", + "dia_id": "D17:6", + "text": "Those places must've been amazing! Nature sure has a way of leaving us speechless." + }, + { + "speaker": "John", + "dia_id": "D17:7", + "text": "Nature sure is powerful and beautiful! It's really humbling to witness such sights." + }, + { + "speaker": "Tim", + "dia_id": "D17:8", + "text": "Yeah! It always makes us realize how huge the world is and how special it is. These moments really show us the beauty around us. Anyways, have you read or watched anything good recently?" + }, + { + "speaker": "John", + "dia_id": "D17:9", + "text": "Yep, I just finished this amazing fantasy series. It was a wild ride with so many twists. The author is amazing at creating awesome storylines and characters - I love getting lost in those fantasy worlds." + }, + { + "speaker": "Tim", + "dia_id": "D17:10", + "text": "That's amazing! Same here. There's something special about being lost in an awesome fantasy realm and seeing what happens. It's like an escape. \"That\" is one of my favorite fantasy shows. Have you seen it?" + }, + { + "speaker": "John", + "dia_id": "D17:11", + "text": "Yeah, I saw \"That\"! It's amazing to see those worlds and characters come alive. It's a great way to escape reality!" + }, + { + "speaker": "Tim", + "dia_id": "D17:12", + "text": "Yeah, it's awesome how books and movies can take you away. A great escape, right?" + }, + { + "speaker": "John", + "dia_id": "D17:13", + "text": "Definitely, it's like a mental break, giving our minds a rest and letting them wander. So refreshing!" + }, + { + "speaker": "Tim", + "dia_id": "D17:14", + "text": "It's like entering another world! We get to take a break from everything and just let our minds wander. It's so nice and refreshing." + }, + { + "speaker": "John", + "dia_id": "D17:15", + "text": "And that's just what we need sometimes." + }, + { + "speaker": "Tim", + "dia_id": "D17:16", + "text": "Taking a break from life can help us recharge and get some peace. Plus, it gives us a chance to reconnect with ourselves and tackle life's challenges with a new outlook." + }, + { + "speaker": "John", + "dia_id": "D17:17", + "text": "Yeah, taking time for ourselves is crucial. It helps us stay sharp and focused. Plus, it helps us gain new perspectives and tackle challenges with more energy. Finding the right balance is key and I'll keep that in mind as I continue my journey." + }, + { + "speaker": "Tim", + "dia_id": "D17:18", + "text": "Balance is key and it varies. Take care of yourself, both mentally and physically, and you'll rock it. You got this, bud!" + }, + { + "speaker": "John", + "dia_id": "D17:19", + "text": "Thanks! Your support means a lot. I'll keep pushing forward. Take care, buddy!" + } + ], + "session_18_date_time": "3:59 pm on 16 November, 2023", + "session_18": [ + { + "speaker": "Tim", + "img_url": [ + "https://static.independent.co.uk/2021/07/28/09/26073722-6212c4b6-62f4-4d1f-b2d9-fab1e4ba16a8.jpg" + ], + "blip_caption": "a photo of a castle with a river running through it", + "query": "castle uk lush greenery", + "dia_id": "D18:1", + "text": "Hey John! Hope you're doing good. Guess what? I went to a castle during my trip to the UK last Friday and it was unbelievable! The architecture and the history were amazing!" + }, + { + "speaker": "John", + "img_url": [ + "https://i.redd.it/plgl67c48sg91.jpg" + ], + "blip_caption": "a photo of a person with a bandage on their leg", + "query": "ankle injury wrapped bandages", + "dia_id": "D18:2", + "text": "Hey Tim! That's awesome! Yeah, it was really cool. Oh man, it's been a tough week for me with this injury. But I'm staying positive. How about you? How's your week been?" + }, + { + "speaker": "Tim", + "dia_id": "D18:3", + "text": "Ouch, bummer about the injury. Hang tight. This week has been swamped with exams for me but I'm plowing through." + }, + { + "speaker": "John", + "blip_caption": "a photo of a notebook with a bunch of notes on it", + "dia_id": "D18:4", + "text": "Cheers, Tim. Injury's been rough, but I'm staying positive. How's the exam prep coming? Confident?" + }, + { + "speaker": "Tim", + "dia_id": "D18:5", + "text": "Exams can be challenging, but I'm putting in my best effort. Feeling optimistic and working diligently! How do you stay motivated during difficult study sessions?" + }, + { + "speaker": "John", + "blip_caption": "a photo of a soccer game with a player on the field", + "dia_id": "D18:6", + "text": "I visualize my goals and success for focus and motivation. It really helps me stay motivated during tough studying. Do you have any study tricks?" + }, + { + "speaker": "Tim", + "dia_id": "D18:7", + "text": "That's cool! I like breaking up my studying into smaller parts. 25 minutes on, then 5 minutes off for something fun. It's less overwhelming and keeps me on track." + }, + { + "speaker": "John", + "dia_id": "D18:8", + "text": "Nice work! Breaking it down into smaller parts is definitely a smart move. I wish you all the best on your exams!" + }, + { + "speaker": "Tim", + "dia_id": "D18:9", + "text": "Thanks! Appreciate your support. I hope your injury heals soon." + }, + { + "speaker": "John", + "dia_id": "D18:10", + "text": "Sure thing, Tim! Got your back. I hope so too. The doctor said it's not too serious." + }, + { + "speaker": "Tim", + "dia_id": "D18:11", + "text": "That's good to hear, I'm glad." + }, + { + "speaker": "John", + "dia_id": "D18:12", + "text": "I hate not being on the court." + }, + { + "speaker": "Tim", + "dia_id": "D18:13", + "text": "I bet. It's like if I couldn't read due to an injury." + }, + { + "speaker": "John", + "dia_id": "D18:14", + "text": "I'm pushing on though. Talk soon!" + }, + { + "speaker": "Tim", + "dia_id": "D18:15", + "text": "Take care! Keep pushing on. Talk soon." + } + ], + "session_19_date_time": "10:22 am on 21 November, 2023", + "session_19": [ + { + "speaker": "Tim", + "dia_id": "D19:1", + "text": "Hey John! Haven't talked in a bit, how ya been? Hope your injury is feeling better." + }, + { + "speaker": "John", + "dia_id": "D19:2", + "text": "Hey Tim! Thanks for checking in. It's been tough, but I'm staying positive and taking it slow. How about you? How have you been?" + }, + { + "speaker": "Tim", + "dia_id": "D19:3", + "text": "I've been swamped with studies and projects, but last week I had a setback. I tried writing a story based on my experiences in the UK, but it didn't go the way I wanted. It's been tough, do you have any advice for getting better with storytelling?" + }, + { + "speaker": "John", + "dia_id": "D19:4", + "text": "Sorry to hear about the setback with your story. I understand how frustrating it can be when things don't go as planned. When I face challenges on the court, I try to reflect on what went wrong and find ways to improve. Maybe you can try doing the same with your storytelling." + }, + { + "speaker": "Tim", + "dia_id": "D19:5", + "text": "Cool idea. Reflecting on what went wrong and how to improve could definitely help me get back on track. Thanks! Out of curiosity, what's been one of your toughest challenges in basketball?" + }, + { + "speaker": "John", + "dia_id": "D19:6", + "text": "Last season, I had a major challenge when I hurt my ankle. It required some time off and physical therapy. It was frustrating because I couldn't play or help the team. I stayed focused on my recovery and worked hard to strengthen my body. It was a tough mental and physical challenge, but it made me realize the importance of patience and perseverance. I'm grateful that I was able to overcome it." + }, + { + "speaker": "Tim", + "dia_id": "D19:7", + "text": "That must have been tough not being able to play and help your team. You did an amazing job staying focused and overcoming it. Your resilience and determination are inspiring! Thanks for sharing." + }, + { + "speaker": "John", + "dia_id": "D19:8", + "text": "Thanks! That means a lot. Difficult times are part of life – what's important is how we handle them. When things get tough, I try to remember why I'm so passionate about basketball. That love and enthusiasm keeps me motivated, no matter what." + }, + { + "speaker": "Tim", + "dia_id": "D19:9", + "text": "When things get tough, it's so important to remember why we love what we do. For me, it's writing and reading. That's what helps me stay motivated and push myself to get better. Has anything similar happened with basketball for you? Tell me about it!" + }, + { + "speaker": "John", + "dia_id": "D19:10", + "text": "I faced some tough times while playing basketball. I messed up during a big game, and it was really hard to accept. Instead of getting stuck in that moment, I worked hard to get better. It taught me that resilience is key and owning up to mistakes is important. Gotta keep growing and striving to be a strong player and teammate. So grateful." + }, + { + "speaker": "Tim", + "dia_id": "D19:11", + "text": "Wow, that's awesome. Admitting mistakes and using them to get better is super important. You really show how much you care about improving. Keep it up!" + }, + { + "speaker": "John", + "dia_id": "D19:12", + "text": "Thanks! I appreciate your support. It's all about growing and getting better, both on and off the court. Let's keep working hard!" + }, + { + "speaker": "Tim", + "img_url": [ + "https://i.redd.it/28n5c71uffu71.jpg" + ], + "blip_caption": "a photo of a bunch of books on a wooden floor", + "query": "fantasy novels movies collection", + "dia_id": "D19:13", + "text": "Yeah, John! Let's keep growing and improving. We got this! These are my companions on my growth journey." + }, + { + "speaker": "John", + "dia_id": "D19:14", + "text": "Fantasy books always fuel my creativity, both in and outside of my hobbies. Are Harry Potter and GoT still your favorites?" + }, + { + "speaker": "Tim", + "dia_id": "D19:15", + "text": "Yes, they are still my favorites - I love how they take me to other places. What other books do you like?" + }, + { + "speaker": "John", + "dia_id": "D19:16", + "text": "I love non-fiction books about personal development and mindset. They help me know myself better. Do you enjoy reading other types of books as well?" + }, + { + "speaker": "Tim", + "dia_id": "D19:17", + "text": "Yep, John! I love getting lost in fantasy stories, but also discovering new ways to better myself through books on growth, psychology, and improving myself. It's wild how much you can learn from them, right?" + }, + { + "speaker": "John", + "dia_id": "D19:18", + "text": "Yeah, Tim! Books really can shift how we think and help us learn totally new things. Have you come across any that made a big impact on you recently?" + }, + { + "speaker": "Tim", + "dia_id": "D19:19", + "text": "Yeah, John! I recently read a book that really made a big impact on me. It's all about how small changes can make big differences. It really changed the way I do things. Have you read any good books lately?" + }, + { + "speaker": "John", + "dia_id": "D19:20", + "text": "I recently finished rereading \"The Alchemist\" - it was really inspiring. It made me think again about following dreams and searching for our own personal legends. I felt really motivated and hopeful after reading it." + }, + { + "speaker": "Tim", + "dia_id": "D19:21", + "text": "Wow, that book is great! I read it a while back and it really changed my perspective on my goals. I'm glad it had the same impact on you!" + }, + { + "speaker": "John", + "dia_id": "D19:22", + "text": "Yeah, that book is really something. It really helped motivate me to keep chasing my dreams and to trust the process. It's amazing how books can have such an impact on us, right?" + }, + { + "speaker": "Tim", + "dia_id": "D19:23", + "text": "Definitely! Books have a way of opening up new worlds, inspiring us, and making us think. They have the power to make us feel better and help us grow, which is amazing. It's great that we share a love for reading. Let's keep exploring books and motivating each other! Talk to you later!" + } + ], + "session_20_date_time": "9:52 am on 1 December, 2023", + "session_20": [ + { + "speaker": "Tim", + "img_url": [ + "https://i.pinimg.com/originals/52/98/a1/5298a13a728c023b77f9cc86529a8748.jpg" + ], + "blip_caption": "a photo of a notepad with a note and pen on it", + "query": "study materials", + "dia_id": "D20:1", + "text": "Hey John! It's been ages since we last chatted. I had a tough exam last week that had me doubting myself. But instead of giving up, I turned it into a learning experience. I studied hard and it showed me how resilient and determined I can be. Here's a pic of my success 👍" + }, + { + "speaker": "John", + "blip_caption": "a photo of a white wall with a black lettering that says 30 positive suites", + "dia_id": "D20:2", + "text": "Hi Tim! Congrats on your success! Keep it up, you're doing great! I'm also trying out yoga to get a little extra strength and flexibility. It's challenging but worth it." + }, + { + "speaker": "Tim", + "dia_id": "D20:3", + "text": "Thanks! I appreciate your encouragement. How's it going with yoga? Have you noticed any improvements?" + }, + { + "speaker": "John", + "dia_id": "D20:4", + "text": "Yoga's been really awesome for me. It's helped me improve in terms of strength and flexibility, as well as focus and balance during my workouts. It's been great!" + }, + { + "speaker": "Tim", + "dia_id": "D20:5", + "text": "Great news! Yoga is indeed amazing for your body and mind. Are there any specific poses that you enjoy practicing?" + }, + { + "speaker": "John", + "dia_id": "D20:6", + "text": "Yeah, there are a couple of poses I really enjoy. Warrior II makes me feel strong and there's one that helps with balance and stability. I love how these poses challenge my body and mind!" + }, + { + "speaker": "Tim", + "dia_id": "D20:7", + "text": "Woohoo! Congrats on finding poses that suit you. Yoga is so cool for showing us what we can really do. Maybe you could share a pic so I can try it too?" + }, + { + "speaker": "John", + "img_url": [ + "https://pixahive.com/wp-content/uploads/2021/02/Virabhadrasana-Warrior-Pose-357219-pixahive.jpg" + ], + "blip_caption": "a photography of a man doing a yoga pose on a blue mat", + "query": "warrior II pose", + "dia_id": "D20:8", + "re-download": true, + "text": "Here's a photo of me in this pose. It's a good way to work out your legs and core. Give it a shot!" + }, + { + "speaker": "Tim", + "dia_id": "D20:9", + "text": "That's a tough one! How long do you usually hold that pose?" + }, + { + "speaker": "John", + "dia_id": "D20:10", + "text": "I typically hold it for 30-60 seconds. It really helps with building strength and stability!" + }, + { + "speaker": "Tim", + "dia_id": "D20:11", + "text": "That's cool, I'm gonna give it a shot and see how it goes. Thanks for the tip!" + }, + { + "speaker": "John", + "dia_id": "D20:12", + "text": "No worries! Let me know how it goes. Happy to help whenever you need it!" + }, + { + "speaker": "Tim", + "dia_id": "D20:13", + "text": "Thanks! Your support and encouragement have truly made this journey better. I really appreciate it." + }, + { + "speaker": "John", + "dia_id": "D20:14", + "text": "I'm here for you. You've got this!" + }, + { + "speaker": "Tim", + "blip_caption": "a photo of a bookcase filled with dvds and games", + "dia_id": "D20:15", + "text": "Thanks! Your support means a lot to me. Your friendship means a lot too." + }, + { + "speaker": "John", + "dia_id": "D20:16", + "text": "Thanks, I really appreciate it. Your friendship means a lot to me too." + }, + { + "speaker": "Tim", + "img_url": [ + "https://i.redd.it/9nvpaptnspc81.jpg" + ], + "blip_caption": "a photo of a book shelf with many books on it", + "query": "fantasy novels game of thrones harry potter bookshelf", + "dia_id": "D20:17", + "text": "Glad we're friends! Plus, bonus points for both being into fantasy books and movies. I just reorganized my book shelf, speaking of." + }, + { + "speaker": "John", + "dia_id": "D20:18", + "text": "Cool! Can I take a closer peek at it? What are some of your favorites?" + }, + { + "speaker": "Tim", + "img_url": [ + "https://i.redd.it/optq3zb96w771.jpg" + ], + "blip_caption": "a photo of a book shelf with a lot of books on it", + "query": "bookshelf harry potter game of thrones lord of the rings", + "dia_id": "D20:19", + "text": "Yeah, check it out - here's my bookshelf! I have some of my favorites on there, like these ones. It's an amazing journey!" + }, + { + "speaker": "John", + "dia_id": "D20:20", + "text": "That bookshelf is awesome! The Hobbit is one of my favorites too. What an amazing journey!" + }, + { + "speaker": "Tim", + "dia_id": "D20:21", + "text": "Glad you like it! The Hobbit is great, but have you read that other popular fantasy series? It's also awesome!" + }, + { + "speaker": "John", + "dia_id": "D20:22", + "text": "Yeah, I've read that other popular fantasy series too! It's one of my favorites. It has such a cool story!" + }, + { + "speaker": "Tim", + "dia_id": "D20:23", + "text": "It's awesome how these books take us to different worlds!" + }, + { + "speaker": "John", + "dia_id": "D20:24", + "text": "It's like escaping to these incredible new worlds and having a break from reality for a fun adventure." + }, + { + "speaker": "Tim", + "dia_id": "D20:25", + "text": "Yeah, that's why I love them. They let us take a break from reality and have an awesome adventure. So magical!" + }, + { + "speaker": "John", + "img_url": [ + "https://i.redd.it/huz7cq7rtix31.jpg" + ], + "blip_caption": "a photo of a forest with sun shining through the trees", + "query": "forest ray of sunlight through trees", + "dia_id": "D20:26", + "text": "Yeah, it's awesome! Like being transported to a different world with all those amazing moments - so fun!" + }, + { + "speaker": "Tim", + "dia_id": "D20:27", + "text": "Wow, what an awesome shot! Feels like a magical forest - where was that?" + }, + { + "speaker": "John", + "dia_id": "D20:28", + "text": "The photo is from a forest near my hometown. It's so tranquil." + }, + { + "speaker": "Tim", + "dia_id": "D20:29", + "text": "Wow, nature's amazing! We're lucky to have places like that near our homes." + }, + { + "speaker": "John", + "dia_id": "D20:30", + "text": "It's incredible how we have these beautiful places near our homes. We should definitely appreciate them." + }, + { + "speaker": "Tim", + "dia_id": "D20:31", + "text": "It really does have a way of calming us and reminding us of the beauty around." + }, + { + "speaker": "John", + "img_url": [ + "https://i.redd.it/5qw4hbg418561.jpg" + ], + "blip_caption": "a photo of a lake with a rock in the middle of it", + "query": "serene lake mountains background", + "dia_id": "D20:32", + "text": "Definitely! It grounds us and makes us appreciate the simple beauty around us. We should take time to enjoy it." + }, + { + "speaker": "Tim", + "dia_id": "D20:33", + "text": "That picture looks super peaceful! It reminds me of a trip I took last summer." + }, + { + "speaker": "John", + "img_url": [ + "https://i.redd.it/lf5a5i0jnah31.jpg" + ], + "blip_caption": "a photo of a campfire with chairs and a lake in the background", + "query": "campfire lake sunset", + "dia_id": "D20:34", + "text": "We had a blast camping and disconnecting from the everyday." + }, + { + "speaker": "Tim", + "dia_id": "D20:35", + "text": "Looks great! Where did you go camping?" + }, + { + "speaker": "John", + "dia_id": "D20:36", + "text": "We went camping in the mountains and it was stunning! The air was so refreshing." + }, + { + "speaker": "Tim", + "dia_id": "D20:37", + "text": "Sounds great! Being in the mountains is the best. What was your favorite part of it?" + }, + { + "speaker": "John", + "dia_id": "D20:38", + "text": "I loved just chilling and taking in the beauty of nature. It was super peaceful and refreshing." + }, + { + "speaker": "Tim", + "img_url": [ + "https://images.pexels.com/photos/16598991/pexels-photo-16598991.jpeg?cs\\u003dsrgb\\u0026dl\\u003dpexels-cheng-cj-16598991.jpg" + ], + "blip_caption": "a photo of a plane flying over a mountain range with snow on the top", + "query": "snowy mountain peak", + "dia_id": "D20:39", + "text": "Yeah, nature has that effect on me too. It's like a reset for the soul." + }, + { + "speaker": "John", + "img_url": [ + "https://i.redd.it/q46xsv4ciu641.jpg" + ], + "blip_caption": "a photo of a mountain with a snow covered peak in the distance", + "query": "plane mountain range snow rocky mountains breathtaking experience majestic peaks fresh air", + "dia_id": "D20:40", + "text": "Yeah, nature's great for clearing the mind and calming the soul. This was my Rocky Mountains trip last year and it was stunning. Seeing those mountains, fresh air - it makes you realize how incredible the world is." + }, + { + "speaker": "Tim", + "img_url": [ + "https://i.redd.it/u51y7czkhfrb1.jpg" + ], + "blip_caption": "a photo of a mountain range with a sunset in the background", + "query": "rocky mountains mountain range sunrise", + "dia_id": "D20:41", + "text": "Wow, this is amazing! Nature is really awesome - it makes us feel tiny but connected." + }, + { + "speaker": "John", + "dia_id": "D20:42", + "text": "Nature does have a way of humbling us and showing us our place in the world. It's truly amazing and comforting." + }, + { + "speaker": "Tim", + "dia_id": "D20:43", + "text": "Yeah. It reminds us that we're not alone - we're part of something bigger. Bye!" + } + ], + "session_21_date_time": "5:34 pm on 6 December, 2023", + "session_21": [ + { + "speaker": "Tim", + "img_url": [ + "https://i.redd.it/w404u5zhn0yz.jpg" + ], + "blip_caption": "a photo of a map of westendell on a wall", + "query": "map pinned destinations", + "dia_id": "D21:1", + "text": "Hey John! Haven't talked in a few days, wanted to let you know I joined a travel club! Always been interested in different cultures and countries and I'm excited to check it out. Can't wait to meet new people and learn about what makes them unique!" + }, + { + "speaker": "John", + "img_url": [ + "https://bloximages.chicago2.vip.townnews.com/syvnews.com/content/tncms/assets/v3/editorial/3/38/338dd0b4-adc1-5a60-97e6-9eb3508b3f77/63c7306e4f912.hires.jpg" + ], + "blip_caption": "a photo of three young men standing next to each other on a basketball court", + "query": "teammates basketball smiling", + "dia_id": "D21:2", + "text": "Hey Tim! That's cool! I love learning about different cultures. It's really cool to meet people with different backgrounds. My teammates come from all over." + }, + { + "speaker": "Tim", + "dia_id": "D21:3", + "text": "Wow! How long have you been playing professionally?" + }, + { + "speaker": "John", + "dia_id": "D21:4", + "text": "I've been playing professionally for just under a year now. It's been a wild ride." + }, + { + "speaker": "Tim", + "dia_id": "D21:5", + "text": "Wow,! Being a pro basketball player must be quite a journey. Is it living up to your expectations?" + }, + { + "speaker": "John", + "dia_id": "D21:6", + "text": "Yeah, it's been great! Challenges, growth, all that jazz—it's been amazing." + }, + { + "speaker": "Tim", + "dia_id": "D21:7", + "text": "Cool! Glad to hear that this journey has been rewarding for you. Could you tell me more about your growth?" + }, + { + "speaker": "John", + "dia_id": "D21:8", + "text": "Yup, on the court, I'm getting better at my overall game. Money-wise, I've gotten some cool endorsement deals. Plus, I'm learning how to market myself and boost my brand. It's been really rewarding to see all these areas progress. What about you? Anything new happening?" + }, + { + "speaker": "Tim", + "dia_id": "D21:9", + "text": "Joined a travel club and, like I said, working on studies. Also picked up new skills. Recently started learning an instrument. Challenging but fun, always admired musicians. Finally giving it a go." + }, + { + "speaker": "John", + "dia_id": "D21:10", + "text": "Learning an instrument is really cool. What instrument are you playing? What genres of music do you want to learn?" + }, + { + "speaker": "Tim", + "img_url": [ + "https://i1.pickpik.com/photos/964/641/216/musical-instrument-violin-curl-tuning-pegs-preview.jpg" + ], + "blip_caption": "a photography of a violin and a violin stick on a sheet of music", + "query": "piano sheet music headphones", + "dia_id": "D21:11", + "re-download": true, + "text": "I'm learning how to play the violin now. I'm mostly into classical music but I'm keen to try out jazz and film scores too. It's a great way to chill and get creative." + }, + { + "speaker": "John", + "dia_id": "D21:12", + "text": "Wow! I hope I can hear you play the violin some day. How long have you been playing the piano again?" + }, + { + "speaker": "Tim", + "dia_id": "D21:13", + "text": "I've been playing for about four months now and it's been an amazing adventure. I'm really enjoying the progress I've been making." + }, + { + "speaker": "John", + "img_url": [ + "https://live.staticflickr.com/7162/6733335339_3a64489025_b.jpg" + ], + "blip_caption": "a photography of a man sitting on the ground with a trophy", + "query": "basketball trophy", + "dia_id": "D21:14", + "re-download": true, + "text": "Nice one! Learning something new is always a great adventure. Keep up the hard work and let's see where you end up. It's all about dedication and effort. It feels great to finally achieve something after putting in so much time and energy." + }, + { + "speaker": "Tim", + "dia_id": "D21:15", + "text": "Congrats on the trophy! It must have felt great to finally get something after putting in so much effort. Do you have any tips on motivating others on your team?" + }, + { + "speaker": "John", + "dia_id": "D21:16", + "text": "Thanks! Winning was awesome. When motivating others, it's important to show care for teammates, celebrate their achievements, provide constructive feedback, and remind them of the bigger goal. Creating a positive environment and giving a pep talk before a game can also be helpful. It's all about supporting and uplifting each other. Do you have any specific strategies in mind?" + }, + { + "speaker": "Tim", + "dia_id": "D21:17", + "text": "Thanks for the helpful advice. Creating a constructive atmosphere and setting an example by working hard can really inspire people. It’s also inspiring to use our own stories to encourage others. Much appreciated!" + }, + { + "speaker": "John", + "dia_id": "D21:18", + "text": "No problem! It's great to use our own experiences to inspire others. Hard work can lead to success. Keep it up! Let me know if you need any assistance." + }, + { + "speaker": "Tim", + "dia_id": "D21:19", + "text": "Thanks! Appreciate the offer. Let me know if you can lend a hand. Bye!" + } + ], + "session_22_date_time": "7:42 pm on 8 December, 2023", + "session_22": [ + { + "speaker": "Tim", + "dia_id": "D22:1", + "text": "Hey John! Long time no see! I just got back from the coolest Harry Potter party. Met lots of awesome people who were into the same stuff as me, had so much fun!" + }, + { + "speaker": "John", + "dia_id": "D22:2", + "text": "Hey Tim! Sounds awesome! So glad you had a blast at the Harry Potter party. Last August I told you about my fun time at a charity event with Harry Potter trivia. Love being with people who are as passionate about Harry Potter as us! Did you dress up as any character?" + }, + { + "speaker": "Tim", + "img_url": [ + "https://dixiedelightsonline.com/wp-content/uploads/2018/05/IMG_4747.jpg" + ], + "blip_caption": "a photo of a chocolate frog in a box on a table", + "query": "gryffindor scarf chocolate frog", + "dia_id": "D22:3", + "text": "It was awesome. I didn't dress as any character, but I wore my Gryffindor scarf. Everyone had cool costumes. I even got this as a treat. Any recent meet-ups with your basketball team?" + }, + { + "speaker": "John", + "img_url": [ + "https://npr.brightspotcdn.com/df/09/e6b1689047eaa3b05e7d61c36c05/image-from-ios-1.jpg" + ], + "blip_caption": "a photo of a group of people riding on top of a fire truck", + "query": "basketball team victory celebration", + "dia_id": "D22:4", + "text": "That frog looks yummy! I haven't had one in ages. Been having some wild games lately, we played a top team and it was tough, but we fought hard and got the win! It's awesome having my team to push us all." + }, + { + "speaker": "Tim", + "dia_id": "D22:5", + "text": "Wow, looks fun! What was the best part for you? And congratulations on the win!" + }, + { + "speaker": "John", + "img_url": [ + "https://live.staticflickr.com/4030/4426181605_d36196a029_c.jpg" + ], + "blip_caption": "a photography of a group of young men sitting on top of a basketball court", + "query": "post-match team huddle", + "dia_id": "D22:6", + "re-download": true, + "text": "Thanks! The best part for me was the camaraderie we built both on and off the court. Winning felt amazing and it was definitely worth all the hard work we put in." + }, + { + "speaker": "Tim", + "img_url": [ + "https://www.thegibsonedge.com/hs-fs/hubfs/images/Blog_Images/Beware%20The%20Person%20Of%20One%20Book%20-%20Flashback%20Friday.jpg" + ], + "blip_caption": "a photo of a stack of books sitting on top of a table", + "query": "fantasy novels stack bookmarks power friendship loyalty", + "dia_id": "D22:7", + "text": "Wow, that's awesome! It's great to see how close you all have become. You must feel a great sense of unity. I'm reading this amazing series about the power of friendship and loyalty – really inspiring stuff. Anything special you do to keep that bond strong?" + }, + { + "speaker": "John", + "img_url": [ + "https://live.staticflickr.com/5128/5297313790_4330145c09_b.jpg" + ], + "blip_caption": "a photography of a group of people sitting around a table eating", + "query": "team dinner basketball games outside of practice", + "dia_id": "D22:8", + "re-download": true, + "text": "Sounds awesome! What kind of stuff do they do in the series? I'm sure the importance of friendship is emphasized. Same with us - we have team dinners, outings, and basketball games. It's those moments away from practice that really build and strengthen our unity." + }, + { + "speaker": "Tim", + "dia_id": "D22:9", + "text": "Awesome! Sounds like your team has something similar to the characters in the series. They rely on each other to push through challenges. By the way, what book are you currently reading? I'm always on the lookout for new reads!" + }, + { + "speaker": "John", + "dia_id": "D22:10", + "text": "Thanks! I'm currently reading a book that I really enjoy. I highly recommend it!" + }, + { + "speaker": "Tim", + "dia_id": "D22:11", + "text": "Sounds cool! Let me know the title so I can add it to my list!" + }, + { + "speaker": "John", + "dia_id": "D22:12", + "text": "I'm reading \"Dune\" by Frank Herbert. It's a great story about religion and human control over ecology. What about you? What's the last book that moved you?" + }, + { + "speaker": "Tim", + "img_url": [ + "https://live.staticflickr.com/4467/24220955708_7548812488_b.jpg" + ], + "blip_caption": "a photography of a book shelf with a book and a book cover", + "query": "the name of the wind book shelf", + "dia_id": "D22:13", + "re-download": true, + "text": "I haven't read that yet but I've heard great things! Just finished \"A Dance with Dragons\" and it's a really good story. Highly recommend it!" + }, + { + "speaker": "John", + "dia_id": "D22:14", + "text": "That's cool! I've heard it's such an inspiring book. Have you read all of George R. R. Martin's books?" + }, + { + "speaker": "Tim", + "dia_id": "D22:15", + "text": "Just the GoT series. Have you tried reading any of them?" + }, + { + "speaker": "John", + "dia_id": "D22:16", + "text": "No, I haven't read them yet but I'll definitely check them out. Cheers!" + }, + { + "speaker": "Tim", + "dia_id": "D22:17", + "text": "Let me know if you get around to them! Have a great day!" + }, + { + "speaker": "John", + "dia_id": "D22:18", + "text": "Thanks! I'll let you know. Have a great day!" + } + ], + "session_23_date_time": "8:28 pm on 11 December, 2023", + "session_23": [ + { + "speaker": "John", + "img_url": [ + "https://assets-global.website-files.com/60ed47e10552352d9d7e0a44/61e5cf5d296761732633ffcc_wsp_banner_w_joanne__dO8vP.jpg" + ], + "blip_caption": "a photo of two women standing next to a banner with sales pros written on it", + "query": "marketing team collaboration trust leadership", + "dia_id": "D23:1", + "text": "Hey Tim, great to see you! Any new success stories?" + }, + { + "speaker": "Tim", + "dia_id": "D23:2", + "text": "Hey John, I had a tough time with my English lit class. Did an analysis on this series and I think it went ok!" + }, + { + "speaker": "John", + "img_url": [ + "https://i.pinimg.com/originals/cf/a0/7f/cfa07fa76047b5d731b3af313d98fc01.jpg" + ], + "blip_caption": "a photo of a basketball game being played in a large arena", + "query": "basketball court game", + "dia_id": "D23:3", + "text": "Thanks! It's a bummer about your English lit class, but you did your best. By the way, I had a career-high in assists last Friday in our big game against our rival. Yay!" + }, + { + "speaker": "Tim", + "dia_id": "D23:4", + "text": "Congrats! That's awesome. How did it feel being out there making those plays?" + }, + { + "speaker": "John", + "dia_id": "D23:5", + "text": "Thanks! It felt great being out there, making plays for my team. I love seeing my teammates succeed because of the opportunities I create for them. The atmosphere in the arena was really electric and playing against our rivals added an extra level of intensity. It was a memorable night!" + }, + { + "speaker": "Tim", + "dia_id": "D23:6", + "text": "Sounds incredible! Must have been quite an atmosphere. Have you had any other games that were as thrilling as this one?" + }, + { + "speaker": "John", + "dia_id": "D23:7", + "text": "I've had some thrilling games in my career. My favorite was when we were down 10 in the 4th and I hit the buzzer-beater shot to win. The atmosphere was incredible and it was such a thrilling experience. Those moments make me love basketball so much." + }, + { + "speaker": "Tim", + "img_url": [ + "https://c8.alamy.com/zooms/9/de5f1d4e73244a8f94b16a6b6d093748/ttnmea.jpg" + ], + "blip_caption": "a photo of a basketball ball on the ground with a basketball hoop in the background", + "query": "basketball court sunset", + "dia_id": "D23:8", + "text": "Wow, John! Moments like that make us love sports, huh? I still think about this pic you sent me a while back." + }, + { + "speaker": "John", + "dia_id": "D23:9", + "text": "Yeah, that pic reminds me of when I was younger. I'd practice basketball outside for hours, dreaming of playing in big games. It was my way of dealing with doubts and stress. It's amazing how a ball and hoop can be so powerful, right?" + }, + { + "speaker": "Tim", + "blip_caption": "a photo of a basketball ball on the ground with a basketball hoop in the background", + "dia_id": "D23:10", + "text": "Yeah! Sports are the best. When we're feeling down, it's a way to express ourselves and stay positive. It's awesome how much basketball has done for you. Keep going with your dreams!" + }, + { + "speaker": "John", + "dia_id": "D23:11", + "text": "Thanks! Appreciate the support. It's been a significant part of my life and allows me to be myself and pursue my passions. Gonna keep chasing my dreams!" + }, + { + "speaker": "Tim", + "dia_id": "D23:12", + "text": "Wow! It's really important to do our own thing and follow our dreams. Keep it up, you're gonna do amazing things!" + }, + { + "speaker": "John", + "dia_id": "D23:13", + "text": "Your encouragement means a lot. Let's keep pushing and following our dreams - we can make a difference!" + }, + { + "speaker": "Tim", + "dia_id": "D23:14", + "text": "Definitely. We both have so much potential! Let's keep supporting each other on our journey towards our dreams." + }, + { + "speaker": "John", + "dia_id": "D23:15", + "text": "Yeah, you're super inspiring and motivating. Keep it up!" + }, + { + "speaker": "Tim", + "dia_id": "D23:16", + "text": "Thanks, it means a lot. Let's keep each other motivated. Bye!" + } + ], + "session_24_date_time": "3:37 pm on 16 December, 2023", + "session_24": [ + { + "speaker": "Tim", + "dia_id": "D24:1", + "text": "Hey John, catch up time! What've you been up to? Any good b-ball games lately?" + }, + { + "speaker": "John", + "img_url": [ + "https://www.salisburypost.com/wp-content/uploads/sites/9/2023/03/Catawba-basketball-01.jpg" + ], + "blip_caption": "a photo of a group of women's basketball players holding up a trophy", + "query": "basketball team celebration", + "dia_id": "D24:2", + "text": "Hey Tim! Nice to talk again. The b-ball games have been crazy. We had a real battle against another team last week. It was close until the final buzzer but we got the win." + }, + { + "speaker": "Tim", + "dia_id": "D24:3", + "text": "Congrats, John! That sounds like an intense game." + }, + { + "speaker": "John", + "dia_id": "D24:4", + "text": "Thanks! We won! It was really close, but we made it!" + }, + { + "speaker": "Tim", + "dia_id": "D24:5", + "text": "Wow, that's amazing! Winning must have been so thrilling!" + }, + { + "speaker": "John", + "dia_id": "D24:6", + "text": "Winning was such a thrill, and it was an awesome moment. These experiences really make me love the game." + }, + { + "speaker": "Tim", + "dia_id": "D24:7", + "text": "You must have been so pumped when it happened! Winning can give us a real confidence boost and makes us keep going with our passions." + }, + { + "speaker": "John", + "img_url": [ + "https://images.fineartamerica.com/images-medium-large-5/basketball-court-sunset-jun-pinzon.jpg" + ], + "blip_caption": "a photography of a basketball hoop in the sunset with a fence", + "query": "basketball court sunset workout", + "dia_id": "D24:8", + "re-download": true, + "text": "Yeah, it really does. It keeps me motivated to keep putting in the effort and makes all the tough times worth it. Here's a pic I took during a morning workout, it's a reminder that the journey can be awesome." + }, + { + "speaker": "Tim", + "dia_id": "D24:9", + "text": "That's a good spot for a morning workout! Can you tell me about some challenges you've faced?" + }, + { + "speaker": "John", + "img_url": [ + "https://i.redd.it/o20k6p31qoe41.jpg" + ], + "blip_caption": "a photo of a person with a cast on their foot", + "query": "sprained ankle crutch wrapped ankle injured ankle", + "dia_id": "D24:10", + "text": "Yeah, I injured myself not too long ago. It sucked because I had to miss some games and couldn't help my team." + }, + { + "speaker": "Tim", + "dia_id": "D24:11", + "text": "Ouch, that's rough. Have you been able to stay active or keep up with your fitness routine while you're recovering?" + }, + { + "speaker": "John", + "img_url": [ + "https://mainephysicaltherapy.com/wp-content/uploads/2017/12/Incline-one-arm-cable-pull-down-1.jpg" + ], + "blip_caption": "a photo of a man sitting on a chair with a blue ball", + "query": "physical therapy exercises", + "dia_id": "D24:12", + "text": "It's been tough, but I'm trying to stay active and do my rehab. I do physical therapy exercises every day." + }, + { + "speaker": "Tim", + "dia_id": "D24:13", + "text": "Cool, rehab can be tough but it's key to keep it up. How's it coming along?" + }, + { + "speaker": "John", + "img_url": [ + "https://live.staticflickr.com/5189/5618665304_b2e0ccd051_b.jpg" + ], + "blip_caption": "a photography of a treadmill in a room with a window", + "query": "treadmill gym", + "dia_id": "D24:14", + "re-download": true, + "text": "It's going great! I've been working hard and it's paying off. Last Friday, I had a milestone moment at the gym. I was able to jog a bit with no pain, which was such a relief!" + }, + { + "speaker": "Tim", + "dia_id": "D24:15", + "text": "Wow! How was it jogging without any discomfort?" + }, + { + "speaker": "John", + "dia_id": "D24:16", + "text": "It was great! After being out for so long, jogging without any pain was a huge success. My wife and I hosted a small get-together with friends and family to celebrate." + }, + { + "speaker": "Tim", + "dia_id": "D24:17", + "text": "Congrats! That's awesome. Keep at it and you'll be back in no time. That sounds fun, how was it?" + }, + { + "speaker": "John", + "dia_id": "D24:18", + "text": "Thanks! Appreciate the support and encouragement. I'm gonna keep pushing and staying positive. It was good to see everyone again! We had a ton of fun." + }, + { + "speaker": "Tim", + "dia_id": "D24:19", + "text": "I'm glad everyone had fun!" + }, + { + "speaker": "John", + "dia_id": "D24:20", + "text": "Me too. Talk to you later!" + } + ], + "session_25_date_time": "10:04 am on 19 December, 2023", + "session_25": [ + { + "speaker": "Tim", + "dia_id": "D25:1", + "text": "Hey John, been a while since we chatted. How's it going?" + }, + { + "speaker": "John", + "img_url": [ + "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQiyDMnkQOmMQNKQfQDDRQlTRpNME7oy0xNlcHjLYB8n_ZwvnWe8nYlRKA3WkZNHEQLsME&usqp=CAU" + ], + "blip_caption": "a photography of a man with a backpack and a backpack walking down a path", + "query": "endorsement deal outdoor gear company photoshoot", + "dia_id": "D25:2", + "re-download": true, + "text": "Yo Tim! Great to hear from you. Things have been wild! Last week I got this amazing deal with a renowned outdoor gear company. So pumped!" + }, + { + "speaker": "Tim", + "dia_id": "D25:3", + "text": "That's awesome about the deal! I'm curious, what kind of gear did you end up getting? And how did the photoshoot turn out?" + }, + { + "speaker": "John", + "dia_id": "D25:4", + "text": "Cheers! Got some awesome hiking stuff and outdoor gear - all top-notch. The photoshoot went really well too. We did it in a gorgeous forest and the photographer got some epic shots of me doing my thing - it was amazing!" + }, + { + "speaker": "Tim", + "dia_id": "D25:5", + "text": "Wow! That sounds amazing. Being out in such a gorgeous location must have been incredible. I'd love to see one of the epic shots you got! Do you have any pictures from the photoshoot?" + }, + { + "speaker": "John", + "img_url": [ + "https://cdn.stocksnap.io/img-thumbs/960w/man-jumping_FOTAMAJTAF.jpg" + ], + "blip_caption": "a photography of a man jumping in the air in a field", + "query": "photoshoot forest leap nature", + "dia_id": "D25:6", + "re-download": true, + "text": "Here you go, here's a pic. Nature puts me in a great mood and always gets me energized!" + }, + { + "speaker": "Tim", + "dia_id": "D25:7", + "text": "That's an amazing photo! I can see why it inspires you - the rocks and river look so peaceful. What drew you to that spot?" + }, + { + "speaker": "John", + "dia_id": "D25:8", + "text": "I stumbled across this spot while hiking. The sound of that river was so soothing, I felt so at peace surrounded by those rocks. It was like nature was telling me to stop and admire its beauty." + }, + { + "speaker": "Tim", + "dia_id": "D25:9", + "text": "Wow, that sounds amazing. It's true, nature has a way of bringing peace and joy. Anything else like that been happening lately?" + }, + { + "speaker": "John", + "dia_id": "D25:10", + "text": "Things have been going great on the court. We've been putting in a lot of work and achieving our goals, which is awesome." + }, + { + "speaker": "Tim", + "dia_id": "D25:11", + "text": "Hard work pays off, right? What have you and your team been up to lately?" + }, + { + "speaker": "John", + "dia_id": "D25:12", + "text": "We gave it our all during last week's scrimmage. It's amazing to see our team's growth. We know it won't be easy, but it'll be worth it when we see the results." + }, + { + "speaker": "Tim", + "dia_id": "D25:13", + "text": "What areas have you seen the most growth in during your training?" + }, + { + "speaker": "John", + "dia_id": "D25:14", + "text": "Our team has seen the most growth in communication and bonding. It has really helped our performances by allowing us to understand each other's strengths and weaknesses." + }, + { + "speaker": "Tim", + "dia_id": "D25:15", + "text": "Wow, that's awesome! Glad to hear you guys are bonding. Keep it up!" + }, + { + "speaker": "John", + "dia_id": "D25:16", + "text": "Thanks! Let's keep at it and continue supporting each other. Appreciate your assistance!" + }, + { + "speaker": "Tim", + "dia_id": "D25:17", + "text": "Yeah, let's support each other. I'm here for you. Just keep believing in yourself! Bye!" + } + ], + "session_26_date_time": "3:35 pm on 26 December, 2023", + "session_26": [ + { + "speaker": "John", + "img_url": [ + "https://pivitu.com/wp-content/uploads/bb-plugin/cache/IMG_8996-circle-5996d578100cbb83f34040d0584d4834-5db4d83b44005.jpg" + ], + "blip_caption": "a photo of a basketball court with a crowd of people watching", + "query": "athletic marketing seminar young athletes", + "dia_id": "D26:1", + "text": "Hey Tim! Great to hear from you. My week's been busy - I started doing seminars, helping people with their sports and marketing. It's been awesome!" + }, + { + "speaker": "Tim", + "dia_id": "D26:2", + "text": "Hey John! Sounds awesome! Congrats on how far you've come. How did it go?" + }, + { + "speaker": "John", + "img_url": [ + "https://i.redd.it/5hvq16lfcgy71.jpg" + ], + "blip_caption": "a photo of a man and woman on stage at a convention", + "query": "seminar speaker stage crowd participants", + "dia_id": "D26:3", + "text": "Thanks! The seminars went really well. All the aspiring profs were so eager and motivated - it was great! I'm really happy I could share my knowledge and help out." + }, + { + "speaker": "Tim", + "blip_caption": "a photo of a book with a golden cover on a table", + "dia_id": "D26:4", + "text": "Wow John! Impressive stuff! I'm starting some big new things too!" + }, + { + "speaker": "John", + "dia_id": "D26:5", + "text": "Thanks! What have you been up to?" + }, + { + "speaker": "Tim", + "img_url": [ + "https://i.redd.it/53yi43izjpb81.jpg" + ], + "blip_caption": "a photo of a book with a picture of a boy and a girl", + "query": "adventures across the globe stories travelers book cover", + "dia_id": "D26:6", + "text": "I've been reading cool stories from travelers from around the world. I'm using it to plan my next adventure. This is a book I found with tons of them!" + }, + { + "speaker": "John", + "dia_id": "D26:7", + "text": "Wow, that's cool! Have you read any of the stories? I'm looking for some travel ideas too." + }, + { + "speaker": "Tim", + "img_url": [ + "https://i.redd.it/tdukekw3wlp91.jpg" + ], + "blip_caption": "a photo of two men on horseback in front of a mountain", + "query": "hiking himalayas snowy mountain peak", + "dia_id": "D26:8", + "text": "I read a few of them. One of them is about two hikers who trekked through the Himalayas, sounds awesome!" + }, + { + "speaker": "John", + "dia_id": "D26:9", + "text": "Wow, that sounds awesome! How challenging was the trek through the Himalayas?" + }, + { + "speaker": "Tim", + "dia_id": "D26:10", + "text": "The book mentioned that the trek was tough but worth it, with challenging terrain, altitude sickness, and bad weather. But they made it and saw amazing sights - it really motivated me." + }, + { + "speaker": "John", + "dia_id": "D26:11", + "text": "Wow! Sounds like a tough journey." + }, + { + "speaker": "Tim", + "dia_id": "D26:12", + "text": "It's true. Facing challenges can be tough, but it can make us stronger. I just visited a travel agency to see what the requirements would be for my next dream trip." + }, + { + "speaker": "John", + "dia_id": "D26:13", + "text": "For sure, challenges help us learn and grow. Sounds fun! Keep me updated!" + }, + { + "speaker": "Tim", + "blip_caption": "a photo of a newspaper article with a picture of a woman", + "dia_id": "D26:14", + "text": "Thanks, I will. You have to keep pushing for your goals." + }, + { + "speaker": "John", + "dia_id": "D26:15", + "text": "By the way, who was that main actress in Harry Potter? I've heard about her a lot lately." + }, + { + "speaker": "Tim", + "dia_id": "D26:16", + "text": "Emma Watson, she's a big supporter of gender equality. I'm a huge fan." + }, + { + "speaker": "John", + "dia_id": "D26:17", + "text": "Wow, that's great! It's inspiring to see people who use their platform for important causes and make a difference." + }, + { + "speaker": "Tim", + "dia_id": "D26:18", + "text": "Her women's rights advocacy is also a huge inspiration to me! Seeing people use their platform for causes like gender equality is really inspiring. It's so cool to see people making a difference." + }, + { + "speaker": "John", + "img_url": [ + "https://talkstar-photos.s3.amazonaws.com/uploads/9dd73626-fe0d-4a3b-b913-c39d65250da8/ApolloRobbins_2013G-embed.jpg" + ], + "blip_caption": "a photography of two men standing next to each other on a stage", + "query": "charity event speaking", + "dia_id": "D26:19", + "re-download": true, + "text": "Definitely. Making a difference is important to me. I use my influence and resources to help causes I believe in. It's about making the world a better place. Here's a picture of me speaking at a charity event." + }, + { + "speaker": "Tim", + "dia_id": "D26:20", + "text": "Cool! What causes are you working on? Tell me more about them!" + }, + { + "speaker": "John", + "img_url": [ + "https://necommunitycenter.org/portland/wp-content/uploads/2023/01/Game-in-Progress-with-Ref2-1024x684.jpg" + ], + "blip_caption": "a photo of a group of kids playing basketball in a gym", + "query": "youth sports programs basketball court kids playing", + "dia_id": "D26:21", + "text": "I've been working on supporting youth sports and fighting for fair chances in sports for underserved communities. It's important to me that every kid has access to good sports programs. I've been collaborating with organizations to create more opportunities for young athletes and help them succeed. It's amazing to see the difference sports make in people's lives." + }, + { + "speaker": "Tim", + "dia_id": "D26:22", + "text": "Cool! What have been some memorable experiences working with them?" + }, + { + "speaker": "John", + "dia_id": "D26:23", + "text": "Organizing a basketball camp for kids in my hometown last summer was an awesome experience! Seeing their faces light up when they hit the court was priceless. It was a week full of laughs, high-fives, and personal growth for us all. That opportunity to inspire those kids and show them just how much potential they have was truly incredible." + }, + { + "speaker": "Tim", + "blip_caption": "a photo of a basketball with a signed autograph on it", + "dia_id": "D26:24", + "text": "Wow! Making a difference to those kids was great! Your passion for helping others is awesome." + }, + { + "speaker": "John", + "dia_id": "D26:25", + "text": "Thanks! I'm really glad I can make a difference. Have you been doing anything new in your free time?" + }, + { + "speaker": "Tim", + "img_url": [ + "https://i.redd.it/oqnovgpaxdv51.jpg" + ], + "blip_caption": "a photo of a collection of harry potter books on a desk", + "query": "harry potter book stack", + "dia_id": "D26:26", + "text": "In my downtime, I still love to get lost in good books, and this series is one of my favorites. It's a magical world to escape to." + }, + { + "speaker": "John", + "dia_id": "D26:27", + "text": "That's awesome! Have you seen all the Harry Potter movies? I'm a fan too!" + }, + { + "speaker": "Tim", + "dia_id": "D26:28", + "text": "Yeah, I have! Watching them and seeing how they compare to the books is awesome. It's amazing to watch the story come alive. Have you seen all of them?" + }, + { + "speaker": "John", + "dia_id": "D26:29", + "text": "I'm a total movie fan! Seeing it all come alive on the big screen is awesome, and a great way to relax." + }, + { + "speaker": "Tim", + "img_url": [ + "https://images.pexels.com/photos/7234395/pexels-photo-7234395.jpeg" + ], + "blip_caption": "a photography of three guys sitting on a couch watching a movie", + "query": "movie night friends popcorn harry potter", + "dia_id": "D26:30", + "re-download": true, + "text": "Yeah, watching movies is a fun way to relax. We love having movie marathons with our friends." + }, + { + "speaker": "John", + "dia_id": "D26:31", + "text": "Sounds like a blast! Movie marathons with friends and popcorn, right? So, what's your favorite genre?" + }, + { + "speaker": "Tim", + "img_url": [ + "https://i.redd.it/ixsrm1ukws611.jpg" + ], + "blip_caption": "a photo of a poster of a group of people with a sword", + "query": "fantasy movie poster lord of the rings", + "dia_id": "D26:32", + "text": "I'm a huge fan of this genre! Epic adventures and magical worlds are my thing. Here's a pic of my favorite, Lord of the Rings!" + }, + { + "speaker": "John", + "dia_id": "D26:33", + "text": "Wow, that's great! Are there any new fantasy movies that you're excited about?" + }, + { + "speaker": "Tim", + "dia_id": "D26:34", + "text": "Woo-hoo! There's a new fantasy TV series coming out next month - can't wait!" + }, + { + "speaker": "John", + "dia_id": "D26:35", + "text": "What's it called? I'm always down for something new." + }, + { + "speaker": "Tim", + "dia_id": "D26:36", + "text": "I'm really excited to watch this new show that's coming out called \"The Wheel of Time\". It's based on a book series that I love." + }, + { + "speaker": "John", + "dia_id": "D26:37", + "text": "That sounds exciting!" + }, + { + "speaker": "Tim", + "dia_id": "D26:38", + "text": "Yeah, can't wait to check out the series. It's always fun seeing the books come to life on screen! Talk to you later!" + } + ], + "session_27_date_time": "5:26 pm on 2 January, 2024", + "session_27": [ + { + "speaker": "Tim", + "blip_caption": "a photo of a man standing on a fence in front of a leaning tower", + "dia_id": "D27:1", + "text": "Hi John, how's it going? Interesting things have happened since we last talked - I joined a group of globetrotters who are into the same stuff as me. It's been awesome getting to know them and hear about their trips." + }, + { + "speaker": "John", + "dia_id": "D27:2", + "text": "Hey Tim! Cool to hear about your globetrotting group! Must be great connecting with other traveling buffs. By the way, have you been to Italy? I had a blast there last month." + }, + { + "speaker": "Tim", + "blip_caption": "a photo of a book with a tag on it", + "dia_id": "D27:3", + "text": "It's been awesome chatting with fellow travel enthusiasts. Italy is definitely on my list of places to visit. How was your trip there last month?" + }, + { + "speaker": "John", + "dia_id": "D27:4", + "text": "Italy was awesome! Everything from the food to the history and architecture was amazing. I even got this awesome book while I was there and it's been giving me some cooking inspiration." + }, + { + "speaker": "Tim", + "dia_id": "D27:5", + "text": "Wow, traveling is amazing, isn't it? I'm learning German now - tough but fun. Do you know any other languages?" + }, + { + "speaker": "John", + "blip_caption": "a photo of a book with a red cover and white title", + "dia_id": "D27:6", + "text": "Wow! Impressive you're learning German. I know a bit of it myself and Spanish, it makes travel so much easier. How's it going with your language studies?\n" + }, + { + "speaker": "Tim", + "blip_caption": "a photo of a book sitting on a table next to a wall", + "dia_id": "D27:7", + "text": "Learning German has been tough but worth it. I like the structure of the language, it's much easier when I took French in high school. What made you decide to learn Spanish?" + }, + { + "speaker": "John", + "dia_id": "D27:8", + "text": "I've always wanted to learn Spanish. I just stared with it. It's such a useful language with many personal and professional opportunities!" + }, + { + "speaker": "Tim", + "img_url": [ + "https://thechinesebujo.files.wordpress.com/2017/03/img_2563.jpg" + ], + "blip_caption": "a photo of a cell phone sitting on a notebook with a smiley face app", + "query": "language learning app on phone", + "dia_id": "D27:9", + "text": "Yeah, knowing another language opens up a lot of opportunities. Have you come across any good resources for learning Spanish? I've been using this app." + }, + { + "speaker": "John", + "dia_id": "D27:10", + "text": "Yeah! I've been using that app on my phone to practice too! It's helped a lot." + }, + { + "speaker": "Tim", + "dia_id": "D27:11", + "text": "That app is great. Learning another language is tough, but the rewards are totally worth it." + }, + { + "speaker": "John", + "blip_caption": "a photo of a basketball ball on the ground with a basketball hoop in the background", + "dia_id": "D27:12", + "text": "It takes dedication and practice, but it's so rewarding to communicate with different cultures. Keep it up with German!" + }, + { + "speaker": "Tim", + "dia_id": "D27:13", + "text": "Thanks! I appreciate your encouragement. I'm definitely going to keep up with my German lessons. Do you still play basketball often?" + }, + { + "speaker": "John", + "img_url": [ + "https://c8.alamy.com/zooms/9/de5f1d4e73244a8f94b16a6b6d093748/ttnmea.jpg" + ], + "blip_caption": "a photo of a basketball ball on the ground with a basketball hoop in the background", + "query": "basketball court sunset", + "dia_id": "D27:14", + "text": "Yeah, basketball is still really important to me - I practice and train every day to stay in shape and improve. Can't imagine my life without it, it's my passion." + }, + { + "speaker": "Tim", + "img_url": [ + "https://i.redd.it/oqnovgpaxdv51.jpg" + ], + "blip_caption": "a photo of a collection of harry potter books on a desk", + "query": "harry potter books stack", + "dia_id": "D27:15", + "text": "Wow! Love the way you go for it. Don't ever quit on what you love. I will always love reading, personally." + }, + { + "speaker": "John", + "dia_id": "D27:16", + "text": "Thanks! I won't give up on it. What got you into books?" + }, + { + "speaker": "Tim", + "img_url": [ + "https://i.pinimg.com/originals/96/80/20/968020204c99f3f30544384d39fe598b.jpg" + ], + "blip_caption": "a photo of a desk with a chair and a book shelf", + "query": "harry potter book collection desk", + "dia_id": "D27:17", + "text": "I love escaping to that world. I have a collection of books that take me there." + }, + { + "speaker": "John", + "dia_id": "D27:18", + "text": "That's awesome! I totally understand why reading means so much to you. It's amazing how much playing a game can help us grow. Thanks for showing us your collection! Which one do you like best that takes you to another world?" + }, + { + "speaker": "Tim", + "blip_caption": "a photo of a collection of movies and dvds on a carpet", + "dia_id": "D27:19", + "text": "Harry Potter is my favorite book. It's so immersive!" + }, + { + "speaker": "John", + "img_url": [ + "https://i.redd.it/ur3tz0poja151.jpg" + ], + "blip_caption": "a photo of a collection of star wars movies on a table", + "query": "fantasy movies dvd collection carpet", + "dia_id": "D27:20", + "text": "Cool! Glad you're enjoying that book! Do you have any favorite fantasy movies as well? These are mine." + }, + { + "speaker": "Tim", + "dia_id": "D27:21", + "text": "Definitely Star Wars! It's my favorite and never gets old. What about you, do you have any favorite fantasy films?" + }, + { + "speaker": "John", + "img_url": [ + "https://i.redd.it/7vtqewbtg1181.jpg" + ], + "blip_caption": "a photo of a shelf with a lot of books on it", + "query": "lord of the rings dvd collection", + "dia_id": "D27:22", + "text": "I'm a huge fan of Lord of the Rings! The adventure, the world, and the characters are awesome." + }, + { + "speaker": "Tim", + "blip_caption": "a photo of a bookcase filled with dvds and games", + "dia_id": "D27:23", + "text": "Wow, me too! That's an awesome collection! Have you watched them heaps? Got any favorite characters from those movies?" + }, + { + "speaker": "John", + "blip_caption": "a photo of a bookmark with a picture of a woman kissing a man", + "dia_id": "D27:24", + "text": "Thanks! I've watched a bunch of them and they're inspiring. My favorite character is Aragorn, he grows so much throughout the story." + }, + { + "speaker": "Tim", + "dia_id": "D27:25", + "text": "Nice one! Why is he your favorite?" + }, + { + "speaker": "John", + "dia_id": "D27:26", + "text": "He's a great leader and puts others first - that's why he eventually becomes king." + }, + { + "speaker": "Tim", + "img_url": [ + "https://i.redd.it/mlffg3ze7gr71.jpg" + ], + "blip_caption": "a photo of a painting of a man with long hair", + "query": "aragorn poster lord of the rings", + "dia_id": "D27:27", + "text": "Wow, Aragorn's story is so inspiring - from a ranger to king of Gondor. It's amazing how he grows and achieves redemption throughout his journey." + }, + { + "speaker": "John", + "img_url": [ + "https://i.redd.it/monip6iq5xm51.jpg" + ], + "blip_caption": "a photo of a painting of a man smoking a cigarette", + "query": "aragorn painting lord of the rings", + "dia_id": "D27:28", + "text": "Yeah. His journey is really inspiring. I have a painting in my room to remind me to stay true and be a leader in everything I do." + }, + { + "speaker": "Tim", + "dia_id": "D27:29", + "text": "Wow, that's awesome! What is it about him that makes him so inspiring for you?" + }, + { + "speaker": "John", + "dia_id": "D27:30", + "text": "Aragorn's brave, selfless, down-to-earth attitude is what inspired me. He never gives up and always stands up for justice." + }, + { + "speaker": "Tim", + "img_url": [ + "https://i.redd.it/or3d42qfg4t91.jpg" + ], + "blip_caption": "a photo of a map of the world on a piece of paper", + "query": "map middle earth fantasy novels world-building intricate details fictional universe", + "dia_id": "D27:31", + "text": "Yeah, he's really inspiring. What's awesome about fantasy books like LOTR is getting lost in another world and seeing all the tiny details." + }, + { + "speaker": "John", + "dia_id": "D27:32", + "text": "Yeah, that's what I'm thinking! Love this map, it really helps you get lost in another world. What's on it?" + }, + { + "speaker": "Tim", + "dia_id": "D27:33", + "text": "It's a map of Middle-earth from LOTR - it's really cool to see all the different realms and regions." + }, + { + "speaker": "John", + "dia_id": "D27:34", + "text": "Wow, that looks awesome! Exploring different lands and regions in fantasy stories is always fun!" + }, + { + "speaker": "Tim", + "dia_id": "D27:35", + "text": "Thanks! It's really cool how fantasy stories allow me to explore other cultures and landscapes, all from the comfort of my home." + }, + { + "speaker": "John", + "img_url": [ + "https://i0.wp.com/stlouispatina.com/wp-content/uploads/2022/10/Copyright-St.-Louis-Patina-2771.jpg" + ], + "blip_caption": "a photo of a person walking down a path in front of the eiffel tower", + "query": "eiffel tower", + "dia_id": "D27:36", + "text": "Yeah! That's why I love traveling - it's a way to learn about different cultures and places." + }, + { + "speaker": "Tim", + "blip_caption": "a photo of a group of people climbing up a stone wall", + "dia_id": "D27:37", + "text": "I love traveling too. That picture is awesome. Have you been to Paris? The Eiffel Tower is so cool!" + }, + { + "speaker": "John", + "img_url": [ + "https://i.redd.it/b9cuf3yfd3s91.jpg" + ], + "blip_caption": "a photo of a view of a city from a bird's eye view", + "query": "eiffel tower view from top", + "dia_id": "D27:38", + "text": "Thanks! Yeah, I've been there before and loved it! That place is amazing and the view from there is incredible!" + }, + { + "speaker": "Tim", + "dia_id": "D27:39", + "text": "Wow, John, it looks amazing! Can't wait to see it for myself. Traveling is so eye-opening!" + }, + { + "speaker": "John", + "dia_id": "D27:40", + "text": "Yeah, it really is. It helps you see new things and get a different view of everything. It's so cool and educational! Talk to you later!" + } + ], + "session_28_date_time": "5:24 pm on 7 January, 2024", + "session_28": [ + { + "speaker": "Tim", + "dia_id": "D28:1", + "text": "Hey John, long time no talk. On Friday, I got great news - I'm finally in the study abroad program I applied for! Next month, I'm off to Ireland for a semester." + }, + { + "speaker": "John", + "dia_id": "D28:2", + "text": "Congrats, Tim! That's amazing news. So, where are you going to stay?" + }, + { + "speaker": "Tim", + "img_url": [ + "https://roadworksbooks.files.wordpress.com/2022/07/img_4242.jpg" + ], + "blip_caption": "a photo of a woman standing on the side of a street", + "query": "galway colorful street arts scene traditional irish music", + "dia_id": "D28:3", + "text": "Thanks! I'm gonna stay in Galway, it's great for its arts and Irish music. This place has such a vibrant atmosphere." + }, + { + "speaker": "John", + "dia_id": "D28:4", + "text": "Awesome, Galway looks amazing! Is there anything in particular that you're keen to check out while you're there?" + }, + { + "speaker": "Tim", + "img_url": [ + "https://images.pexels.com/photos/8456767/pexels-photo-8456767.jpeg?cs\\u003dsrgb\\u0026dl\\u003dpexels-david-riand-8456767.jpg" + ], + "blip_caption": "a photo of a cliff overlooking the ocean at sunset", + "query": "cliffs moher sunset", + "dia_id": "D28:5", + "text": "Yep! I'm so excited to explore the nature - it looks amazing!" + }, + { + "speaker": "John", + "dia_id": "D28:6", + "text": "Wow, great view! Have you visited any other places?" + }, + { + "speaker": "Tim", + "img_url": [ + "https://i.redd.it/t98kzzhuocp81.jpg" + ], + "blip_caption": "a photo of a person standing on a cliff overlooking the ocean", + "query": "cliffs moher ocean cliffs", + "dia_id": "D28:7", + "text": "I want to visit The Cliffs of Moher. It has amazing ocean views and awesome cliffs." + }, + { + "speaker": "John", + "dia_id": "D28:8", + "text": "Wow, that's awesome! I could stop by there after my season." + }, + { + "speaker": "Tim", + "dia_id": "D28:9", + "text": "Cool, let me know if you're around!" + }, + { + "speaker": "John", + "img_url": [ + "https://outandaboutmummy.files.wordpress.com/2019/11/img_6609.jpg" + ], + "blip_caption": "a photo of a boy dribbling a basketball on a court", + "query": "charity basketball tournament children", + "dia_id": "D28:10", + "text": "Yep, I'll let you know. Oh, I held a benefit basketball game last week." + }, + { + "speaker": "Tim", + "dia_id": "D28:11", + "text": "Wow! How did the game go?" + }, + { + "speaker": "John", + "dia_id": "D28:12", + "text": "The game turned out to be a total success! Lots of people showed up and had a great time, plus we were able to raise some money for charity." + }, + { + "speaker": "Tim", + "dia_id": "D28:13", + "text": "Great job organizing the event. That's really making a difference!" + }, + { + "speaker": "John", + "dia_id": "D28:14", + "text": "Thanks! It's amazing how basketball brings people together and creates a positive impact!" + }, + { + "speaker": "Tim", + "blip_caption": "a photo of a basketball ball and a book on a court", + "dia_id": "D28:15", + "text": "You rock! Keep it up!" + }, + { + "speaker": "John", + "dia_id": "D28:16", + "text": "Thanks, Tim! It's awesome to see how sports can unite people. By the way, what book are you currently reading?" + }, + { + "speaker": "Tim", + "dia_id": "D28:17", + "text": "I'm currently reading a fantasy novel called \"The Name of the Wind\" by Patrick Rothfuss. It's really good!" + }, + { + "speaker": "John", + "dia_id": "D28:18", + "text": "\"The Name of the Wind\" sounds cool. I'll add it to my list. Thanks!" + }, + { + "speaker": "Tim", + "dia_id": "D28:19", + "text": "I hope you enjoy it! Let me know your thoughts." + }, + { + "speaker": "John", + "dia_id": "D28:20", + "text": "Will do! Thanks for the recommendation!" + }, + { + "speaker": "Tim", + "dia_id": "D28:21", + "text": "No problem. Talk to you soon!" + } + ], + "session_29_date_time": "1:41 pm on 12 January, 2024", + "session_29": [ + { + "speaker": "Tim", + "dia_id": "D29:1", + "text": "Hey John! How's it going? Hope all is good." + }, + { + "speaker": "John", + "dia_id": "D29:2", + "text": "Hey Tim! Things have been good. Something exciting happened recently for me. What about you? How's everything going?" + }, + { + "speaker": "Tim", + "dia_id": "D29:3", + "text": "Cool news! I'm trying to get my head around the visa requirements for some places I want to visit. It's kind of overwhelming but I'm excited! What have you been up to?" + }, + { + "speaker": "John", + "img_url": [ + "https://i.redd.it/g33orfsja8ha1.jpg" + ], + "blip_caption": "a photo of a baseball player holding a bat next to a soda", + "query": "endorsement drink logo", + "dia_id": "D29:4", + "text": "Last week was wild - something incredible happened. But it's a total dream come true - just crazy! I got an endorsement with a popular beverage company!" + }, + { + "speaker": "Tim", + "dia_id": "D29:5", + "text": "Congrats! How did it feel to seal the deal?" + }, + { + "speaker": "John", + "dia_id": "D29:6", + "text": "Thanks! It felt crazy. It's not just about the signing, but it's about feeling like all the hard work paid off - like all those training hours weren't for nothing." + }, + { + "speaker": "Tim", + "dia_id": "D29:7", + "text": "Wow! I bet you were thrilled when everything finally worked out. That sense of accomplishment is awesome and really boosts your self-esteem. I can imagine all the hard work you put into it was definitely worth it." + }, + { + "speaker": "John", + "dia_id": "D29:8", + "text": "Yeah, it's great when you reach a goal and it feels rewarding. It's a reminder that you're going in the right direction, and all the hard work was worth it. What's something you feel proud of recently?" + }, + { + "speaker": "Tim", + "dia_id": "D29:9", + "text": "I'm proud of researching visa requirements for countries I want to visit. It feels like taking initiative is a step towards making my travel dreams a reality!" + }, + { + "speaker": "John", + "dia_id": "D29:10", + "text": "Great going! Taking initiative is a must if you wanna achieve your goals. I'm excited to hear about all your future adventures!" + }, + { + "speaker": "Tim", + "dia_id": "D29:11", + "text": "Thanks! I'll keep you in the loop about my travels. Is there anywhere you recommend visiting?" + }, + { + "speaker": "John", + "dia_id": "D29:12", + "text": "Barcelona is a must-visit city! You'll love exploring the culture, admiring the architecture, and tasting the amazing food in each neighborhood. Plus, the nearby beaches are great for soaking up the sun. Definitely add it to your travel list!" + }, + { + "speaker": "Tim", + "dia_id": "D29:13", + "text": "Barcelona sounds awesome! I've heard so many great things. Definitely adding it to my list. Thanks!" + }, + { + "speaker": "John", + "dia_id": "D29:14", + "text": "No problem! Glad you liked the suggestion. Let me know if you have any other questions or need help with anything." + }, + { + "speaker": "Tim", + "dia_id": "D29:15", + "text": "Cheers! I owe you one. Let me know if you need anything. Bye!" + } + ] + }, + "qa": [ + { + "question": "What fantasy movies does Tim like?", + "answer": "Lord of the Rings, Harry Potter, and Star Wars.", + "evidence": [ + "D8:16", + "D8:18", + "D26:28", + "D26:32", + "D27:21" + ], + "category": 1 + }, + { + "question": "Which US states might Tim be in during September 2023 based on his plans of visiting Universal Studios?", + "answer": "California or Florida", + "evidence": [ + "D10:9" + ], + "category": 3 + }, + { + "question": "How did John overcome a mistake he made during a big game in basketball?", + "answer": "Worked hard to get better and focused on growth", + "evidence": [ + "D19:10" + ], + "category": 4 + }, + { + "question": "What is a Star Wars book that Tim might enjoy?", + "answer": "Star Wars: Jedi Apprentice by Judy Blundell and David Farland. It is a highly rated and immersive series about his favorite movies.", + "evidence": [ + "D27:19", + "D27:21" + ], + "category": 3 + }, + { + "question": "Why do Tim and John find LeBron inspiring?", + "answer": "LeBron's determination and the epic block in Game 7 of the '16 Finals", + "evidence": [ + "D16:9", + "D16:10" + ], + "category": 4 + }, + { + "question": "Which popular time management technique does Tim use to prepare for exams?", + "answer": "Pomodoro technique", + "evidence": [ + "D18:3", + "D18:7" + ], + "category": 3 + }, + { + "question": "What has John cooked?", + "answer": "Soup, a slow cooker meal, and honey garlic chicken with roasted veg.", + "evidence": [ + "D10:4", + "D15:30", + "D15:31", + "D15:32" + ], + "category": 1 + }, + { + "question": "Would Tim enjoy reading books by C. S. Lewis or John Greene?", + "answer": "C. S.Lewis", + "evidence": [ + "D1:14", + "D1:16", + "D1:18" + ], + "category": 3 + }, + { + "question": "What kind of fiction stories does Tim write?", + "answer": "Fantasy stories with plot twists", + "evidence": [ + "D15:3", + "D16:1" + ], + "category": 1 + }, + { + "question": "What does John do to share his knowledge?", + "answer": "gives seminars, mentors younger players.", + "evidence": [ + "D14:3", + "D26:1" + ], + "category": 1 + }, + { + "question": "When did Tim start playing the violin?", + "answer": "August 2023", + "evidence": [ + "D21:13" + ], + "category": 2 + }, + { + "question": "What kind of yoga for building core strength might John benefit from?", + "answer": "Hatha Yoga", + "evidence": [ + "D20:2" + ], + "category": 3 + }, + { + "question": "What could John do after his basketball career?", + "answer": "become a basketball coach since he likes giving back and leadership", + "evidence": [ + "D11:19", + "D26:1", + "D27:26" + ], + "category": 3 + }, + { + "question": "Where are John and his teammates planning to explore on a team trip?", + "answer": "a new city", + "evidence": [ + "D11:7" + ], + "category": 4 + }, + { + "question": "Which month was John in Italy?", + "answer": "December, 2023", + "evidence": [ + "D27:2" + ], + "category": 2 + }, + { + "question": "What did John's teammates give him when they met on Aug 15th?", + "answer": "a basketball with autographs on it", + "evidence": [ + "D7:7" + ], + "category": 4 + }, + { + "question": "What new fantasy TV series is Tim excited about?", + "answer": "\"The Wheel of Time\"", + "evidence": [ + "D26:36" + ], + "category": 4 + }, + { + "question": "Which US cities does John mention visiting to Tim?", + "answer": "Seattle, Chicago, New York", + "evidence": [ + "D3:19", + "D6:3", + "D9:6" + ], + "category": 1 + }, + { + "question": "How did John describe the views during their road trip out on the European coastline?", + "answer": "Spectacular", + "evidence": [ + "D17:3" + ], + "category": 4 + }, + { + "question": "How long did John and his high school basketball teammates play together?", + "answer": "Four years", + "evidence": [ + "D9:4" + ], + "category": 4 + }, + { + "question": "Which Star Wars-related locations would Tim enjoy during his visit to Ireland?", + "answer": "Skellig Michael, Malin Head, Loop Head, Ceann Sibéal, and Brow Head because they are Star Wars filming locations.", + "evidence": [ + "D1:18", + "D27:21", + "D28:1" + ], + "category": 3 + }, + { + "question": "What book did Tim just finish reading on 8th December, 2023?", + "answer": "\"A Dance with Dragons\"", + "evidence": [ + "D22:13" + ], + "category": 4 + }, + { + "question": "What books has John read?", + "answer": "inpsiring book on dreaming big, The Alchemist, fantasy series, non-fiction books on personal development, Dune", + "evidence": [ + "D4:10", + "D11:26", + "D17:9", + "D19:16", + "D19:20", + "D22:12" + ], + "category": 1 + }, + { + "question": "How will John share the honey garlic chicken recipe with the other person?", + "answer": "write it down and mail it", + "evidence": [ + "D15:34" + ], + "category": 4 + }, + { + "question": "When did John attend the Harry Potter trivia?", + "answer": "August 2023.", + "evidence": [ + "D4:8", + "D22:2" + ], + "category": 2 + }, + { + "question": "What outdoor activities does John enjoy?", + "answer": "Hiking, surfing", + "evidence": [ + "D3:27", + "D12:6" + ], + "category": 1 + } + ] + } +] \ No newline at end of file diff --git a/benchmark/RAG/datasets/Benchmark_Lite/Qasper/qasper-dev-v0.3.json b/benchmark/RAG/datasets/Benchmark_Lite/Qasper/qasper-dev-v0.3.json new file mode 100644 index 00000000..8ed5ab72 --- /dev/null +++ b/benchmark/RAG/datasets/Benchmark_Lite/Qasper/qasper-dev-v0.3.json @@ -0,0 +1,5158 @@ +{ + "1904.04358": { + "title": "Deep Learning the EEG Manifold for Phonological Categorization from Active Thoughts", + "abstract": "Speech-related Brain Computer Interfaces (BCI) aim primarily at finding an alternative vocal communication pathway for people with speaking disabilities. As a step towards full decoding of imagined speech from active thoughts, we present a BCI system for subject-independent classification of phonological categories exploiting a novel deep learning based hierarchical feature extraction scheme. To better capture the complex representation of high-dimensional electroencephalography (EEG) data, we compute the joint variability of EEG electrodes into a channel cross-covariance matrix. We then extract the spatio-temporal information encoded within the matrix using a mixed deep neural network strategy. Our model framework is composed of a convolutional neural network (CNN), a long-short term network (LSTM), and a deep autoencoder. We train the individual networks hierarchically, feeding their combined outputs in a final gradient boosting classification step. Our best models achieve an average accuracy of 77.9% across five different binary classification tasks, providing a significant 22.5% improvement over previous methods. As we also show visually, our work demonstrates that the speech imagery EEG possesses significant discriminative information about the intended articulatory movements responsible for natural speech synthesis.", + "full_text": [ + { + "section_name": "Introduction", + "paragraphs": [ + "Decoding intended speech or motor activity from brain signals is one of the major research areas in Brain Computer Interface (BCI) systems BIBREF0 , BIBREF1 . In particular, speech-related BCI technologies attempt to provide effective vocal communication strategies for controlling external devices through speech commands interpreted from brain signals BIBREF2 . Not only do they provide neuro-prosthetic help for people with speaking disabilities and neuro-muscular disorders like locked-in-syndrome, nasopharyngeal cancer, and amytotropic lateral sclerosis (ALS), but also equip people with a better medium to communicate and express thoughts, thereby improving the quality of rehabilitation and clinical neurology BIBREF3 , BIBREF4 . Such devices also have applications in entertainment, preventive treatments, personal communication, games, etc. Furthermore, BCI technologies can be utilized in silent communication, as in noisy environments, or situations where any sort of audio-visual communication is infeasible.", + "Among the various brain activity-monitoring modalities in BCI, electroencephalography (EEG) BIBREF5 , BIBREF6 has demonstrated promising potential to differentiate between various brain activities through measurement of related electric fields. EEG is non-invasive, portable, low cost, and provides satisfactory temporal resolution. This makes EEG suitable to realize BCI systems. EEG data, however, is challenging: these data are high dimensional, have poor SNR, and suffer from low spatial resolution and a multitude of artifacts. For these reasons, it is not particularly obvious how to decode the desired information from raw EEG signals. Although the area of BCI based speech intent recognition has received increasing attention among the research community in the past few years, most research has focused on classification of individual speech categories in terms of discrete vowels, phonemes and words BIBREF7 , BIBREF8 , BIBREF9 , BIBREF10 , BIBREF11 , BIBREF12 , BIBREF13 , BIBREF14 , BIBREF15 . This includes categorization of imagined EEG signal into binary vowel categories like /a/, /u/ and rest BIBREF7 , BIBREF8 , BIBREF9 ; binary syllable classes like /ba/ and /ku/ BIBREF1 , BIBREF10 , BIBREF11 , BIBREF12 ; a handful of control words like 'up', 'down', 'left', 'right' and 'select' BIBREF15 or others like 'water', 'help', 'thanks', 'food', 'stop' BIBREF13 , Chinese characters BIBREF14 , etc. Such works mostly involve traditional signal processing or manual feature handcrafting along with linear classifiers (e.g., SVMs). In our recent work BIBREF16 , we introduced deep learning models for classification of vowels and words that achieved 23.45% improvement of accuracy over the baseline.", + "Production of articulatory speech is an extremely complicated process, thereby rendering understanding of the discriminative EEG manifold corresponding to imagined speech highly challenging. As a result, most of the existing approaches failed to achieve satisfactory accuracy on decoding speech tokens from the speech imagery EEG data. Perhaps, for these reasons, very little work has been devoted to relating the brain signals to the underlying articulation. The few exceptions include BIBREF17 , BIBREF18 . In BIBREF17 , Zhao et al. used manually handcrafted features from EEG data, combined with speech audio and facial features to achieve classification of the phonological categories varying based on the articulatory steps. However, the imagined speech classification accuracy based on EEG data alone, as reported in BIBREF17 , BIBREF18 , are not satisfactory in terms of accuracy and reliability. We now turn to describing our proposed models." + ] + }, + { + "section_name": "Proposed Framework", + "paragraphs": [ + "Cognitive learning process underlying articulatory speech production involves incorporation of intermediate feedback loops and utilization of past information stored in the form of memory as well as hierarchical combination of several feature extractors. To this end, we develop our mixed neural network architecture composed of three supervised and a single unsupervised learning step, discussed in the next subsections and shown in Fig. FIGREF1 . We formulate the problem of categorizing EEG data based on speech imagery as a non-linear mapping INLINEFORM0 of a multivariate time-series input sequence INLINEFORM1 to fixed output INLINEFORM2 , i.e, mathematically INLINEFORM3 : INLINEFORM4 , where c and t denote the EEG channels and time instants respectively." + ] + }, + { + "section_name": "Preprocessing step", + "paragraphs": [ + "We follow similar pre-processing steps on raw EEG data as reported in BIBREF17 (ocular artifact removal using blind source separation, bandpass filtering and subtracting mean value from each channel) except that we do not perform Laplacian filtering step since such high-pass filtering may decrease information content from the signals in the selected bandwidth." + ] + }, + { + "section_name": "Joint variability of electrodes", + "paragraphs": [ + "Multichannel EEG data is high dimensional multivariate time series data whose dimensionality depends on the number of electrodes. It is a major hurdle to optimally encode information from these EEG data into lower dimensional space. In fact, our investigation based on a development set (as we explain later) showed that well-known deep neural networks (e.g., fully connected networks such as convolutional neural networks, recurrent neural networks and autoencoders) fail to individually learn such complex feature representations from single-trial EEG data. Besides, we found that instead of using the raw multi-channel high-dimensional EEG requiring large training times and resource requirements, it is advantageous to first reduce its dimensionality by capturing the information transfer among the electrodes. Instead of the conventional approach of selecting a handful of channels as BIBREF17 , BIBREF18 , we address this by computing the channel cross-covariance, resulting in positive, semi-definite matrices encoding the connectivity of the electrodes. We define channel cross-covariance (CCV) between any two electrodes INLINEFORM0 and INLINEFORM1 as: INLINEFORM2 . Next, we reject the channels which have significantly lower cross-covariance than auto-covariance values (where auto-covariance implies CCV on same electrode). We found this measure to be essential as the higher cognitive processes underlying speech planning and synthesis involve frequent information exchange between different parts of the brain. Hence, such matrices often contain more discriminative features and hidden information than mere raw signals. This is essentially different than our previous work BIBREF16 where we extract per-channel 1-D covariance information and feed it to the networks. We present our sample 2-D EEG cross-covariance matrices (of two individuals) in Fig. FIGREF2 ." + ] + }, + { + "section_name": "CNN & LSTM", + "paragraphs": [ + "In order to decode spatial connections between the electrodes from the channel covariance matrix, we use a CNN BIBREF19 , in particular a four-layered 2D CNN stacking two convolutional and two fully connected hidden layers. The INLINEFORM0 feature map at a given CNN layer with input INLINEFORM1 , weight matrix INLINEFORM2 and bias INLINEFORM3 is obtained as: INLINEFORM4 . At this first level of hierarchy, the network is trained with the corresponding labels as target outputs, optimizing a cross-entropy cost function. In parallel, we apply a four-layered recurrent neural network on the channel covariance matrices to explore the hidden temporal features of the electrodes. Namely, we exploit an LSTM BIBREF20 consisting of two fully connected hidden layers, stacked with two LSTM layers and trained in a similar manner as CNN." + ] + }, + { + "section_name": "Deep autoencoder for spatio-temporal information", + "paragraphs": [ + "As we found the individually-trained parallel networks (CNN and LSTM) to be useful (see Table TABREF12 ), we suspected the combination of these two networks could provide a more powerful discriminative spatial and temporal representation of the data than each independent network. As such, we concatenate the last fully-connected layer from the CNN with its counterpart in the LSTM to compose a single feature vector based on these two penultimate layers. Ultimately, this forms a joint spatio-temporal encoding of the cross-covariance matrix.", + "In order to further reduce the dimensionality of the spatio-temporal encodings and cancel background noise effects BIBREF21 , we train an unsupervised deep autoenoder (DAE) on the fused heterogeneous features produced by the combined CNN and LSTM information. The DAE forms our second level of hierarchy, with 3 encoding and 3 decoding layers, and mean squared error (MSE) as the cost function." + ] + }, + { + "section_name": "Classification with Extreme Gradient Boost", + "paragraphs": [ + "At the third level of hierarchy, the discrete latent vector representation of the deep autoencoder is fed into an Extreme Gradient Boost based classification layer BIBREF22 , BIBREF23 motivated by BIBREF21 . It is a regularized gradient boosted decision tree that performs well on structured problems. Since our EEG-phonological pairwise classification has an internal structure involving individual phonemes and words, it seems to be a reasonable choice of classifier. The classifier receives its input from the latent vectors of the deep autoencoder and is trained in a supervised manner to output the final predicted classes corresponding to the speech imagery." + ] + }, + { + "section_name": "Dataset", + "paragraphs": [ + "We evaluate our model on a publicly available dataset, KARA ONE BIBREF17 , composed of multimodal data for stimulus-based, imagined and articulated speech state corresponding to 7 phonemic/syllabic ( /iy/, /piy/, /tiy/, /diy/, /uw/, /m/, /n/ ) as well as 4 words(pat, pot, knew and gnaw). The dataset consists of 14 participants, with each prompt presented 11 times to each individual. Since our intention is to classify the phonological categories from human thoughts, we discard the facial and audio information and only consider the EEG data corresponding to imagined speech. It is noteworthy that given the mixed nature of EEG signals, it is reportedly challenging to attain a pairwise EEG-phoneme mapping BIBREF18 . In order to explore the problem space, we thus specifically target five binary classification problems addressed in BIBREF17 , BIBREF18 , i.e presence/absence of consonants, phonemic nasal, bilabial, high-front vowels and high-back vowels." + ] + }, + { + "section_name": "Training and hyperparameter selection", + "paragraphs": [ + "We performed two sets of experiments with the single-trial EEG data. In PHASE-ONE, our goals was to identify the best architectures and hyperparameters for our networks with a reasonable number of runs. For PHASE-ONE, we randomly shuffled and divided the data (1913 signals from 14 individuals) into train (80%), development (10%) and test sets (10%). In PHASE-TWO, in order to perform a fair comparison with the previous methods reported on the same dataset, we perform a leave-one-subject out cross-validation experiment using the best settings we learn from PHASE-ONE.", + "The architectural parameters and hyperparameters listed in Table TABREF6 were selected through an exhaustive grid-search based on the validation set of PHASE-ONE. We conducted a series of empirical studies starting from single hidden-layered networks for each of the blocks and, based on the validation accuracy, we increased the depth of each given network and selected the optimal parametric set from all possible combinations of parameters. For the gradient boosting classification, we fixed the maximum depth at 10, number of estimators at 5000, learning rate at 0.1, regularization coefficient at 0.3, subsample ratio at 0.8, and column-sample/iteration at 0.4. We did not find any notable change of accuracy while varying other hyperparameters while training gradient boost classifier." + ] + }, + { + "section_name": "Performance analysis and discussion", + "paragraphs": [ + "To demonstrate the significance of the hierarchical CNN-LSTM-DAE method, we conducted separate experiments with the individual networks in PHASE-ONE of experiments and summarized the results in Table TABREF12 From the average accuracy scores, we observe that the mixed network performs much better than individual blocks which is in agreement with the findings in BIBREF21 . A detailed analysis on repeated runs further shows that in most of the cases, LSTM alone does not perform better than chance. CNN, on the other hand, is heavily biased towards the class label which sees more training data corresponding to it. Though the situation improves with combined CNN-LSTM, our analysis clearly shows the necessity of a better encoding scheme to utilize the combined features rather than mere concatenation of the penultimate features of both networks.", + "The very fact that our combined network improves the classification accuracy by a mean margin of 14.45% than the CNN-LSTM network indeed reveals that the autoencoder contributes towards filtering out the unrelated and noisy features from the concatenated penultimate feature set. It also proves that the combined supervised and unsupervised neural networks, trained hierarchically, can learn the discriminative manifold better than the individual networks and it is crucial for improving the classification accuracy. In addition to accuracy, we also provide the kappa coefficients BIBREF24 of our method in Fig. FIGREF14 . Here, a higher mean kappa value corresponding to a task implies that the network is able to find better discriminative information from the EEG data beyond random decisions. The maximum above-chance accuracy (75.92%) is recorded for presence/absence of the vowel task and the minimum (49.14%) is recorded for the INLINEFORM0 .", + "To further investigate the feature representation achieved by our model, we plot T-distributed Stochastic Neighbor Embedding (tSNE) corresponding to INLINEFORM0 and V/C classification tasks in Fig. FIGREF8 . We particularly select these two tasks as our model exhibits respectively minimum and maximum performance for these two. The tSNE visualization reveals that the second set of features are more easily separable than the first one, thereby giving a rationale for our performance.", + "Next, we provide performance comparison of the proposed approach with the baseline methods for PHASE-TWO of our study (cross-validation experiment) in Table TABREF15 . Since the model encounters the unseen data of a new subject for testing, and given the high inter-subject variability of the EEG data, a reduction in the accuracy was expected. However, our network still managed to achieve an improvement of 18.91, 9.95, 67.15, 2.83 and 13.70 % over BIBREF17 . Besides, our best model shows more reliability compared to previous works: The standard deviation of our model's classification accuracy across all the tasks is reduced from 22.59% BIBREF17 and 17.52% BIBREF18 to a mere 5.41%." + ] + }, + { + "section_name": "Conclusion and future direction", + "paragraphs": [ + "In an attempt to move a step towards understanding the speech information encoded in brain signals, we developed a novel mixed deep neural network scheme for a number of binary classification tasks from speech imagery EEG data. Unlike previous approaches which mostly deal with subject-dependent classification of EEG into discrete vowel or word labels, this work investigates a subject-invariant mapping of EEG data with different phonological categories, varying widely in terms of underlying articulator motions (eg: involvement or non-involvement of lips and velum, variation of tongue movements etc). Our model takes an advantage of feature extraction capability of CNN, LSTM as well as the deep learning benefit of deep autoencoders. We took BIBREF17 , BIBREF18 as the baseline works investigating the same problem and compared our performance with theirs. Our proposed method highly outperforms the existing methods across all the five binary classification tasks by a large average margin of 22.51%." + ] + }, + { + "section_name": "Acknowledgments", + "paragraphs": [ + "This work was funded by the Natural Sciences and Engineering Research Council (NSERC) of Canada and Canadian Institutes for Health Research (CIHR)." + ] + } + ], + "qas": [ + { + "question": "How do they demonstrate that this type of EEG has discriminative information about the intended articulatory movements responsible for speech?", + "question_id": "7ae38f51243cb80b16a1df14872b72a1f8a2048f", + "nlp_background": "five", + "topic_background": "", + "paper_read": "", + "search_query": "", + "question_writer": "e8b24c3133e0bec0a6465e1f13acd3a5ed816b37", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "we plot T-distributed Stochastic Neighbor Embedding (tSNE) corresponding to INLINEFORM0 and V/C classification tasks in Fig. FIGREF8 ." + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "To further investigate the feature representation achieved by our model, we plot T-distributed Stochastic Neighbor Embedding (tSNE) corresponding to INLINEFORM0 and V/C classification tasks in Fig. FIGREF8 . We particularly select these two tasks as our model exhibits respectively minimum and maximum performance for these two. The tSNE visualization reveals that the second set of features are more easily separable than the first one, thereby giving a rationale for our performance.", + "FLOAT SELECTED: Fig. 3. tSNE feature visualization for ±nasal (left) and V/C classification (right). Red and green colours indicate the distribution of two different types of features" + ], + "highlighted_evidence": [ + "To further investigate the feature representation achieved by our model, we plot T-distributed Stochastic Neighbor Embedding (tSNE) corresponding to INLINEFORM0 and V/C classification tasks in Fig. FIGREF8 . We particularly select these two tasks as our model exhibits respectively minimum and maximum performance for these two. The tSNE visualization reveals that the second set of features are more easily separable than the first one, thereby giving a rationale for our performance.", + "FLOAT SELECTED: Fig. 3. tSNE feature visualization for ±nasal (left) and V/C classification (right). Red and green colours indicate the distribution of two different types of features" + ] + }, + "annotation_id": "5653e01e666e6542de7df9102aa4f7ffeed12e96", + "worker_id": "a0b403873302db7cada39008f04d01155ef68f4f" + } + ] + }, + { + "question": "What are the five different binary classification tasks?", + "question_id": "deb89bca0925657e0f91ab5daca78b9e548de2bd", + "nlp_background": "five", + "topic_background": "", + "paper_read": "", + "search_query": "", + "question_writer": "e8b24c3133e0bec0a6465e1f13acd3a5ed816b37", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + " presence/absence of consonants, phonemic nasal, bilabial, high-front vowels and high-back vowels." + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "We evaluate our model on a publicly available dataset, KARA ONE BIBREF17 , composed of multimodal data for stimulus-based, imagined and articulated speech state corresponding to 7 phonemic/syllabic ( /iy/, /piy/, /tiy/, /diy/, /uw/, /m/, /n/ ) as well as 4 words(pat, pot, knew and gnaw). The dataset consists of 14 participants, with each prompt presented 11 times to each individual. Since our intention is to classify the phonological categories from human thoughts, we discard the facial and audio information and only consider the EEG data corresponding to imagined speech. It is noteworthy that given the mixed nature of EEG signals, it is reportedly challenging to attain a pairwise EEG-phoneme mapping BIBREF18 . In order to explore the problem space, we thus specifically target five binary classification problems addressed in BIBREF17 , BIBREF18 , i.e presence/absence of consonants, phonemic nasal, bilabial, high-front vowels and high-back vowels." + ], + "highlighted_evidence": [ + " In order to explore the problem space, we thus specifically target five binary classification problems addressed in BIBREF17 , BIBREF18 , i.e presence/absence of consonants, phonemic nasal, bilabial, high-front vowels and high-back vowels." + ] + }, + "annotation_id": "5aaf5c697ef418c87f89954633318d9fed2ef1cc", + "worker_id": "a0b403873302db7cada39008f04d01155ef68f4f" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "presence/absence of consonants, presence/absence of phonemic nasal, presence/absence of bilabial, presence/absence of high-front vowels, and presence/absence of high-back vowels", + "evidence": [ + "We evaluate our model on a publicly available dataset, KARA ONE BIBREF17 , composed of multimodal data for stimulus-based, imagined and articulated speech state corresponding to 7 phonemic/syllabic ( /iy/, /piy/, /tiy/, /diy/, /uw/, /m/, /n/ ) as well as 4 words(pat, pot, knew and gnaw). The dataset consists of 14 participants, with each prompt presented 11 times to each individual. Since our intention is to classify the phonological categories from human thoughts, we discard the facial and audio information and only consider the EEG data corresponding to imagined speech. It is noteworthy that given the mixed nature of EEG signals, it is reportedly challenging to attain a pairwise EEG-phoneme mapping BIBREF18 . In order to explore the problem space, we thus specifically target five binary classification problems addressed in BIBREF17 , BIBREF18 , i.e presence/absence of consonants, phonemic nasal, bilabial, high-front vowels and high-back vowels." + ], + "highlighted_evidence": [ + "In order to explore the problem space, we thus specifically target five binary classification problems addressed in BIBREF17 , BIBREF18 , i.e presence/absence of consonants, phonemic nasal, bilabial, high-front vowels and high-back vowels." + ] + }, + "annotation_id": "ec6b4edbdeef73c6352c5a37971964a0b2fbe16c", + "worker_id": "c1018a31c3272ce74964a3280069f62f314a1a58" + } + ] + }, + { + "question": "How was the spatial aspect of the EEG signal computed?", + "question_id": "9c33b340aefbc1f15b6eb6fb3e23ee615ce5b570", + "nlp_background": "five", + "topic_background": "", + "paper_read": "", + "search_query": "", + "question_writer": "e8b24c3133e0bec0a6465e1f13acd3a5ed816b37", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "we use a CNN BIBREF19 , in particular a four-layered 2D CNN stacking two convolutional and two fully connected hidden layers." + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "In order to decode spatial connections between the electrodes from the channel covariance matrix, we use a CNN BIBREF19 , in particular a four-layered 2D CNN stacking two convolutional and two fully connected hidden layers. The INLINEFORM0 feature map at a given CNN layer with input INLINEFORM1 , weight matrix INLINEFORM2 and bias INLINEFORM3 is obtained as: INLINEFORM4 . At this first level of hierarchy, the network is trained with the corresponding labels as target outputs, optimizing a cross-entropy cost function. In parallel, we apply a four-layered recurrent neural network on the channel covariance matrices to explore the hidden temporal features of the electrodes. Namely, we exploit an LSTM BIBREF20 consisting of two fully connected hidden layers, stacked with two LSTM layers and trained in a similar manner as CNN." + ], + "highlighted_evidence": [ + "In order to decode spatial connections between the electrodes from the channel covariance matrix, we use a CNN BIBREF19 , in particular a four-layered 2D CNN stacking two convolutional and two fully connected hidden layers. The INLINEFORM0 feature map at a given CNN layer with input INLINEFORM1 , weight matrix INLINEFORM2 and bias INLINEFORM3 is obtained as: INLINEFORM4 . At this first level of hierarchy, the network is trained with the corresponding labels as target outputs, optimizing a cross-entropy cost function. In parallel, we apply a four-layered recurrent neural network on the channel covariance matrices to explore the hidden temporal features of the electrodes. Namely, we exploit an LSTM BIBREF20 consisting of two fully connected hidden layers, stacked with two LSTM layers and trained in a similar manner as CNN." + ] + }, + "annotation_id": "1160e2d2db43330e00e762c21291704dbe6476ee", + "worker_id": "a0b403873302db7cada39008f04d01155ef68f4f" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "They use four-layered 2D CNN and two fully connected hidden layers on the channel covariance matrix to compute the spatial aspect.", + "evidence": [ + "In order to decode spatial connections between the electrodes from the channel covariance matrix, we use a CNN BIBREF19 , in particular a four-layered 2D CNN stacking two convolutional and two fully connected hidden layers. The INLINEFORM0 feature map at a given CNN layer with input INLINEFORM1 , weight matrix INLINEFORM2 and bias INLINEFORM3 is obtained as: INLINEFORM4 . At this first level of hierarchy, the network is trained with the corresponding labels as target outputs, optimizing a cross-entropy cost function. In parallel, we apply a four-layered recurrent neural network on the channel covariance matrices to explore the hidden temporal features of the electrodes. Namely, we exploit an LSTM BIBREF20 consisting of two fully connected hidden layers, stacked with two LSTM layers and trained in a similar manner as CNN." + ], + "highlighted_evidence": [ + "In order to decode spatial connections between the electrodes from the channel covariance matrix, we use a CNN BIBREF19 , in particular a four-layered 2D CNN stacking two convolutional and two fully connected hidden layers." + ] + }, + "annotation_id": "bae0c8f68085f310c95afab35e071659e4dbe051", + "worker_id": "c1018a31c3272ce74964a3280069f62f314a1a58" + } + ] + }, + { + "question": "What data was presented to the subjects to elicit event-related responses?", + "question_id": "e6583c60b13b87fc37af75ffc975e7e316d4f4e0", + "nlp_background": "five", + "topic_background": "", + "paper_read": "", + "search_query": "", + "question_writer": "e8b24c3133e0bec0a6465e1f13acd3a5ed816b37", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "7 phonemic/syllabic ( /iy/, /piy/, /tiy/, /diy/, /uw/, /m/, /n/ ) as well as 4 words(pat, pot, knew and gnaw)" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "We evaluate our model on a publicly available dataset, KARA ONE BIBREF17 , composed of multimodal data for stimulus-based, imagined and articulated speech state corresponding to 7 phonemic/syllabic ( /iy/, /piy/, /tiy/, /diy/, /uw/, /m/, /n/ ) as well as 4 words(pat, pot, knew and gnaw). The dataset consists of 14 participants, with each prompt presented 11 times to each individual. Since our intention is to classify the phonological categories from human thoughts, we discard the facial and audio information and only consider the EEG data corresponding to imagined speech. It is noteworthy that given the mixed nature of EEG signals, it is reportedly challenging to attain a pairwise EEG-phoneme mapping BIBREF18 . In order to explore the problem space, we thus specifically target five binary classification problems addressed in BIBREF17 , BIBREF18 , i.e presence/absence of consonants, phonemic nasal, bilabial, high-front vowels and high-back vowels." + ], + "highlighted_evidence": [ + "We evaluate our model on a publicly available dataset, KARA ONE BIBREF17 , composed of multimodal data for stimulus-based, imagined and articulated speech state corresponding to 7 phonemic/syllabic ( /iy/, /piy/, /tiy/, /diy/, /uw/, /m/, /n/ ) as well as 4 words(pat, pot, knew and gnaw). The dataset consists of 14 participants, with each prompt presented 11 times to each individual. " + ] + }, + "annotation_id": "14e904a72999de9da963f54a303a874b5b6f47ab", + "worker_id": "c1018a31c3272ce74964a3280069f62f314a1a58" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "KARA ONE BIBREF17 , composed of multimodal data for stimulus-based, imagined and articulated speech state corresponding to 7 phonemic/syllabic ( /iy/, /piy/, /tiy/, /diy/, /uw/, /m/, /n/ ) as well as 4 words(pat, pot, knew and gnaw)" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "We evaluate our model on a publicly available dataset, KARA ONE BIBREF17 , composed of multimodal data for stimulus-based, imagined and articulated speech state corresponding to 7 phonemic/syllabic ( /iy/, /piy/, /tiy/, /diy/, /uw/, /m/, /n/ ) as well as 4 words(pat, pot, knew and gnaw). The dataset consists of 14 participants, with each prompt presented 11 times to each individual. Since our intention is to classify the phonological categories from human thoughts, we discard the facial and audio information and only consider the EEG data corresponding to imagined speech. It is noteworthy that given the mixed nature of EEG signals, it is reportedly challenging to attain a pairwise EEG-phoneme mapping BIBREF18 . In order to explore the problem space, we thus specifically target five binary classification problems addressed in BIBREF17 , BIBREF18 , i.e presence/absence of consonants, phonemic nasal, bilabial, high-front vowels and high-back vowels." + ], + "highlighted_evidence": [ + "We evaluate our model on a publicly available dataset, KARA ONE BIBREF17 , composed of multimodal data for stimulus-based, imagined and articulated speech state corresponding to 7 phonemic/syllabic ( /iy/, /piy/, /tiy/, /diy/, /uw/, /m/, /n/ ) as well as 4 words(pat, pot, knew and gnaw). The dataset consists of 14 participants, with each prompt presented 11 times to each individual. Since our intention is to classify the phonological categories from human thoughts, we discard the facial and audio information and only consider the EEG data corresponding to imagined speech. It is noteworthy that given the mixed nature of EEG signals, it is reportedly challenging to attain a pairwise EEG-phoneme mapping BIBREF18 . In order to explore the problem space, we thus specifically target five binary classification problems addressed in BIBREF17 , BIBREF18 , i.e presence/absence of consonants, phonemic nasal, bilabial, high-front vowels and high-back vowels." + ] + }, + "annotation_id": "be0701277a315b150b51262e0cce2f91de050198", + "worker_id": "a0b403873302db7cada39008f04d01155ef68f4f" + } + ] + }, + { + "question": "How many electrodes were used on the subject in EEG sessions?", + "question_id": "c7b6e6cb997de1660fd24d31759fe6bb21c7863f", + "nlp_background": "five", + "topic_background": "", + "paper_read": "", + "search_query": "", + "question_writer": "e8b24c3133e0bec0a6465e1f13acd3a5ed816b37", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "1913 signals" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "We performed two sets of experiments with the single-trial EEG data. In PHASE-ONE, our goals was to identify the best architectures and hyperparameters for our networks with a reasonable number of runs. For PHASE-ONE, we randomly shuffled and divided the data (1913 signals from 14 individuals) into train (80%), development (10%) and test sets (10%). In PHASE-TWO, in order to perform a fair comparison with the previous methods reported on the same dataset, we perform a leave-one-subject out cross-validation experiment using the best settings we learn from PHASE-ONE." + ], + "highlighted_evidence": [ + " For PHASE-ONE, we randomly shuffled and divided the data (1913 signals from 14 individuals) into train (80%), development (10%) and test sets (10%). In PHASE-TWO, in order to perform a fair comparison with the previous methods reported on the same dataset, we perform a leave-one-subject out cross-validation experiment using the best settings we learn from PHASE-ONE." + ] + }, + "annotation_id": "46e3d7e687351c7477f698c838f8de74b97cc116", + "worker_id": "a0b403873302db7cada39008f04d01155ef68f4f" + }, + { + "answer": { + "unanswerable": true, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "", + "evidence": [], + "highlighted_evidence": [] + }, + "annotation_id": "8c46ed7f22ff24fbaf10735a8fedd74f31e7c6cb", + "worker_id": "c1018a31c3272ce74964a3280069f62f314a1a58" + } + ] + }, + { + "question": "How many subjects does the EEG data come from?", + "question_id": "f9f59c171531c452bd2767dc332dc74cadee5120", + "nlp_background": "five", + "topic_background": "", + "paper_read": "", + "search_query": "", + "question_writer": "e8b24c3133e0bec0a6465e1f13acd3a5ed816b37", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "14" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "We evaluate our model on a publicly available dataset, KARA ONE BIBREF17 , composed of multimodal data for stimulus-based, imagined and articulated speech state corresponding to 7 phonemic/syllabic ( /iy/, /piy/, /tiy/, /diy/, /uw/, /m/, /n/ ) as well as 4 words(pat, pot, knew and gnaw). The dataset consists of 14 participants, with each prompt presented 11 times to each individual. Since our intention is to classify the phonological categories from human thoughts, we discard the facial and audio information and only consider the EEG data corresponding to imagined speech. It is noteworthy that given the mixed nature of EEG signals, it is reportedly challenging to attain a pairwise EEG-phoneme mapping BIBREF18 . In order to explore the problem space, we thus specifically target five binary classification problems addressed in BIBREF17 , BIBREF18 , i.e presence/absence of consonants, phonemic nasal, bilabial, high-front vowels and high-back vowels." + ], + "highlighted_evidence": [ + "We evaluate our model on a publicly available dataset, KARA ONE BIBREF17 , composed of multimodal data for stimulus-based, imagined and articulated speech state corresponding to 7 phonemic/syllabic ( /iy/, /piy/, /tiy/, /diy/, /uw/, /m/, /n/ ) as well as 4 words(pat, pot, knew and gnaw). The dataset consists of 14 participants, with each prompt presented 11 times to each individual. " + ] + }, + "annotation_id": "3f9aee5ab2bacd7dabf81d34d7218c8231444999", + "worker_id": "c1018a31c3272ce74964a3280069f62f314a1a58" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "14 participants" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "We evaluate our model on a publicly available dataset, KARA ONE BIBREF17 , composed of multimodal data for stimulus-based, imagined and articulated speech state corresponding to 7 phonemic/syllabic ( /iy/, /piy/, /tiy/, /diy/, /uw/, /m/, /n/ ) as well as 4 words(pat, pot, knew and gnaw). The dataset consists of 14 participants, with each prompt presented 11 times to each individual. Since our intention is to classify the phonological categories from human thoughts, we discard the facial and audio information and only consider the EEG data corresponding to imagined speech. It is noteworthy that given the mixed nature of EEG signals, it is reportedly challenging to attain a pairwise EEG-phoneme mapping BIBREF18 . In order to explore the problem space, we thus specifically target five binary classification problems addressed in BIBREF17 , BIBREF18 , i.e presence/absence of consonants, phonemic nasal, bilabial, high-front vowels and high-back vowels." + ], + "highlighted_evidence": [ + "The dataset consists of 14 participants, with each prompt presented 11 times to each individual. " + ] + }, + "annotation_id": "f1eac4cc0226f4b54073e37d6185159339e46720", + "worker_id": "a0b403873302db7cada39008f04d01155ef68f4f" + } + ] + } + ], + "figures_and_tables": [ + { + "file": "2-Figure2-1.png", + "caption": "Fig. 2. Cross covariance Matrices : Rows correspond to two different subjects; Columns (from left to right) correspond to sample examples for bilabial, nasal, vowel, /uw/, and /iy/." + }, + { + "file": "3-Table1-1.png", + "caption": "Table 1. Selected parameter sets" + }, + { + "file": "3-Figure3-1.png", + "caption": "Fig. 3. tSNE feature visualization for ±nasal (left) and V/C classification (right). Red and green colours indicate the distribution of two different types of features" + }, + { + "file": "4-Table2-1.png", + "caption": "Table 2. Results in accuracy on 10% test data in the first study" + }, + { + "file": "4-Figure4-1.png", + "caption": "Fig. 4. Kappa coefficient values for above-chance accuracy based on Table 2" + }, + { + "file": "4-Table3-1.png", + "caption": "Table 3. Comparison of classification accuracy" + } + ] + }, + "1911.07555": { + "title": "Short Text Language Identification for Under Resourced Languages", + "abstract": "The paper presents a hierarchical naive Bayesian and lexicon based classifier for short text language identification (LID) useful for under resourced languages. The algorithm is evaluated on short pieces of text for the 11 official South African languages some of which are similar languages. The algorithm is compared to recent approaches using test sets from previous works on South African languages as well as the Discriminating between Similar Languages (DSL) shared tasks' datasets. Remaining research opportunities and pressing concerns in evaluating and comparing LID approaches are also discussed.", + "full_text": [ + { + "section_name": "Introduction", + "paragraphs": [ + "Accurate language identification (LID) is the first step in many natural language processing and machine comprehension pipelines. If the language of a piece of text is known then the appropriate downstream models like parts of speech taggers and language models can be applied as required.", + "LID is further also an important step in harvesting scarce language resources. Harvested data can be used to bootstrap more accurate LID models and in doing so continually improve the quality of the harvested data. Availability of data is still one of the big roadblocks for applying data driven approaches like supervised machine learning in developing countries.", + "Having 11 official languages of South Africa has lead to initiatives (discussed in the next section) that have had positive effect on the availability of language resources for research. However, many of the South African languages are still under resourced from the point of view of building data driven models for machine comprehension and process automation.", + "Table TABREF2 shows the percentages of first language speakers for each of the official languages of South Africa. These are four conjunctively written Nguni languages (zul, xho, nbl, ssw), Afrikaans (afr) and English (eng), three disjunctively written Sotho languages (nso, sot, tsn), as well as tshiVenda (ven) and Xitsonga (tso). The Nguni languages are similar to each other and harder to distinguish. The same is true of the Sotho languages.", + "This paper presents a hierarchical naive Bayesian and lexicon based classifier for LID of short pieces of text of 15-20 characters long. The algorithm is evaluated against recent approaches using existing test sets from previous works on South African languages as well as the Discriminating between Similar Languages (DSL) 2015 and 2017 shared tasks.", + "Section SECREF2 reviews existing works on the topic and summarises the remaining research problems. Section SECREF3 of the paper discusses the proposed algorithm and Section SECREF4 presents comparative results." + ] + }, + { + "section_name": "Related Works", + "paragraphs": [ + "The focus of this section is on recently published datasets and LID research applicable to the South African context. An in depth survey of algorithms, features, datasets, shared tasks and evaluation methods may be found in BIBREF0.", + "The datasets for the DSL 2015 & DSL 2017 shared tasks BIBREF1 are often used in LID benchmarks and also available on Kaggle . The DSL datasets, like other LID datasets, consists of text sentences labelled by language. The 2017 dataset, for example, contains 14 languages over 6 language groups with 18000 training samples and 1000 testing samples per language.", + "The recently published JW300 parallel corpus BIBREF2 covers over 300 languages with around 100 thousand parallel sentences per language pair on average. In South Africa, a multilingual corpus of academic texts produced by university students with different mother tongues is being developed BIBREF3. The WiLI-2018 benchmark dataset BIBREF4 for monolingual written natural language identification includes around 1000 paragraphs of 235 languages. A possibly useful link can also be made BIBREF5 between Native Language Identification (NLI) (determining the native language of the author of a text) and Language Variety Identification (LVI) (classification of different varieties of a single language) which opens up more datasets. The Leipzig Corpora Collection BIBREF6, the Universal Declaration of Human Rights and Tatoeba are also often used sources of data.", + "The NCHLT text corpora BIBREF7 is likely a good starting point for a shared LID task dataset for the South African languages BIBREF8. The NCHLT text corpora contains enough data to have 3500 training samples and 600 testing samples of 300+ character sentences per language. Researchers have recently started applying existing algorithms for tasks like neural machine translation in earnest to such South African language datasets BIBREF9.", + "Existing NLP datasets, models and services BIBREF10 are available for South African languages. These include an LID algorithm BIBREF11 that uses a character level n-gram language model. Multiple papers have shown that 'shallow' naive Bayes classifiers BIBREF12, BIBREF8, BIBREF13, BIBREF14, SVMs BIBREF15 and similar models work very well for doing LID. The DSL 2017 paper BIBREF1, for example, gives an overview of the solutions of all of the teams that competed on the shared task and the winning approach BIBREF16 used an SVM with character n-gram, parts of speech tag features and some other engineered features. The winning approach for DSL 2015 used an ensemble naive Bayes classifier. The fasttext classifier BIBREF17 is perhaps one of the best known efficient 'shallow' text classifiers that have been used for LID .", + "Multiple papers have proposed hierarchical stacked classifiers (including lexicons) that would for example first classify a piece of text by language group and then by exact language BIBREF18, BIBREF19, BIBREF8, BIBREF0. Some work has also been done on classifying surnames between Tshivenda, Xitsonga and Sepedi BIBREF20. Additionally, data augmentation BIBREF21 and adversarial training BIBREF22 approaches are potentially very useful to reduce the requirement for data.", + "Researchers have investigated deeper LID models like bidirectional recurrent neural networks BIBREF23 or ensembles of recurrent neural networks BIBREF24. The latter is reported to achieve 95.12% in the DSL 2015 shared task. In these models text features can include character and word n-grams as well as informative character and word-level features learnt BIBREF25 from the training data. The neural methods seem to work well in tasks where more training data is available.", + "In summary, LID of short texts, informal styles and similar languages remains a difficult problem which is actively being researched. Increased confusion can in general be expected between shorter pieces of text and languages that are more closely related. Shallow methods still seem to work well compared to deeper models for LID. Other remaining research opportunities seem to be data harvesting, building standardised datasets and creating shared tasks for South Africa and Africa. Support for language codes that include more languages seems to be growing and discoverability of research is improving with more survey papers coming out. Paywalls also seem to no longer be a problem; the references used in this paper was either openly published or available as preprint papers." + ] + }, + { + "section_name": "Methodology", + "paragraphs": [ + "The proposed LID algorithm builds on the work in BIBREF8 and BIBREF26. We apply a naive Bayesian classifier with character (2, 4 & 6)-grams, word unigram and word bigram features with a hierarchical lexicon based classifier.", + "The naive Bayesian classifier is trained to predict the specific language label of a piece of text, but used to first classify text as belonging to either the Nguni family, the Sotho family, English, Afrikaans, Xitsonga or Tshivenda. The scikit-learn multinomial naive Bayes classifier is used for the implementation with an alpha smoothing value of 0.01 and hashed text features.", + "The lexicon based classifier is then used to predict the specific language within a language group. For the South African languages this is done for the Nguni and Sotho groups. If the lexicon prediction of the specific language has high confidence then its result is used as the final label else the naive Bayesian classifier's specific language prediction is used as the final result. The lexicon is built over all the data and therefore includes the vocabulary from both the training and testing sets.", + "The lexicon based classifier is designed to trade higher precision for lower recall. The proposed implementation is considered confident if the number of words from the winning language is at least one more than the number of words considered to be from the language scored in second place.", + "The stacked classifier is tested against three public LID implementations BIBREF17, BIBREF23, BIBREF8. The LID implementation described in BIBREF17 is available on GitHub and is trained and tested according to a post on the fasttext blog. Character (5-6)-gram features with 16 dimensional vectors worked the best. The implementation discussed in BIBREF23 is available from https://github.com/tomkocmi/LanideNN. Following the instructions for an OSX pip install of an old r0.8 release of TensorFlow, the LanideNN code could be executed in Python 3.7.4. Settings were left at their defaults and a learning rate of 0.001 was used followed by a refinement with learning rate of 0.0001. Only one code modification was applied to return the results from a method that previously just printed to screen. The LID algorithm described in BIBREF8 is also available on GitHub.", + "The stacked classifier is also tested against the results reported for four other algorithms BIBREF16, BIBREF26, BIBREF24, BIBREF15. All the comparisons are done using the NCHLT BIBREF7, DSL 2015 BIBREF19 and DSL 2017 BIBREF1 datasets discussed in Section SECREF2." + ] + }, + { + "section_name": "Results and Analysis", + "paragraphs": [ + "The average classification accuracy results are summarised in Table TABREF9. The accuracies reported are for classifying a piece of text by its specific language label. Classifying text only by language group or family is a much easier task as reported in BIBREF8.", + "Different variations of the proposed classifier were evaluated. A single NB classifier (NB), a stack of two NB classifiers (NB+NB), a stack of a NB classifier and lexicon (NB+Lex) and a lexicon (Lex) by itself. A lexicon with a 50% training token dropout is also listed to show the impact of the lexicon support on the accuracy.", + "From the results it seems that the DSL 2017 task might be harder than the DSL 2015 and NCHLT tasks. Also, the results for the implementation discussed in BIBREF23 might seem low, but the results reported in that paper is generated on longer pieces of text so lower scores on the shorter pieces of text derived from the NCHLT corpora is expected.", + "The accuracy of the proposed algorithm seems to be dependent on the support of the lexicon. Without a good lexicon a non-stacked naive Bayesian classifier might even perform better.", + "The execution performance of some of the LID implementations are shown in Table TABREF10. Results were generated on an early 2015 13-inch Retina MacBook Pro with a 2.9 GHz CPU (Turbo Boosted to 3.4 GHz) and 8GB RAM. The C++ implementation in BIBREF17 is the fastest. The implementation in BIBREF8 makes use of un-hashed feature representations which causes it to be slower than the proposed sklearn implementation. The execution performance of BIBREF23 might improve by a factor of five to ten when executed on a GPU." + ] + }, + { + "section_name": "Conclusion", + "paragraphs": [ + "LID of short texts, informal styles and similar languages remains a difficult problem which is actively being researched. The proposed algorithm was evaluated on three existing datasets and compared to the implementations of three public LID implementations as well as to reported results of four other algorithms. It performed well relative to the other methods beating their results. However, the performance is dependent on the support of the lexicon.", + "We would like to investigate the value of a lexicon in a production system and how to possibly maintain it using self-supervised learning. We are investigating the application of deeper language models some of which have been used in more recent DSL shared tasks. We would also like to investigate data augmentation strategies to reduce the amount of training data that is required.", + "Further research opportunities include data harvesting, building standardised datasets and shared tasks for South Africa as well as the rest of Africa. In general, the support for language codes that include more languages seems to be growing, discoverability of research is improving and paywalls seem to no longer be a big problem in getting access to published research." + ] + } + ], + "qas": [ + { + "question": "What is the approach of previous work?", + "question_id": "012b8a89aea27485797373adbcda32f16f9d7b54", + "nlp_background": "two", + "topic_background": "familiar", + "paper_read": "no", + "search_query": "", + "question_writer": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "'shallow' naive Bayes", + "SVM", + "hierarchical stacked classifiers", + "bidirectional recurrent neural networks" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "Existing NLP datasets, models and services BIBREF10 are available for South African languages. These include an LID algorithm BIBREF11 that uses a character level n-gram language model. Multiple papers have shown that 'shallow' naive Bayes classifiers BIBREF12, BIBREF8, BIBREF13, BIBREF14, SVMs BIBREF15 and similar models work very well for doing LID. The DSL 2017 paper BIBREF1, for example, gives an overview of the solutions of all of the teams that competed on the shared task and the winning approach BIBREF16 used an SVM with character n-gram, parts of speech tag features and some other engineered features. The winning approach for DSL 2015 used an ensemble naive Bayes classifier. The fasttext classifier BIBREF17 is perhaps one of the best known efficient 'shallow' text classifiers that have been used for LID .", + "Multiple papers have proposed hierarchical stacked classifiers (including lexicons) that would for example first classify a piece of text by language group and then by exact language BIBREF18, BIBREF19, BIBREF8, BIBREF0. Some work has also been done on classifying surnames between Tshivenda, Xitsonga and Sepedi BIBREF20. Additionally, data augmentation BIBREF21 and adversarial training BIBREF22 approaches are potentially very useful to reduce the requirement for data.", + "Researchers have investigated deeper LID models like bidirectional recurrent neural networks BIBREF23 or ensembles of recurrent neural networks BIBREF24. The latter is reported to achieve 95.12% in the DSL 2015 shared task. In these models text features can include character and word n-grams as well as informative character and word-level features learnt BIBREF25 from the training data. The neural methods seem to work well in tasks where more training data is available." + ], + "highlighted_evidence": [ + "Multiple papers have shown that 'shallow' naive Bayes classifiers BIBREF12, BIBREF8, BIBREF13, BIBREF14, SVMs BIBREF15 and similar models work very well for doing LID. The DSL 2017 paper BIBREF1, for example, gives an overview of the solutions of all of the teams that competed on the shared task and the winning approach BIBREF16 used an SVM with character n-gram, parts of speech tag features and some other engineered features. The winning approach for DSL 2015 used an ensemble naive Bayes classifier. The fasttext classifier BIBREF17 is perhaps one of the best known efficient 'shallow' text classifiers that have been used for LID .", + "Multiple papers have proposed hierarchical stacked classifiers (including lexicons) that would for example first classify a piece of text by language group and then by exact language BIBREF18, BIBREF19, BIBREF8, BIBREF0.", + "Researchers have investigated deeper LID models li" + ] + }, + "annotation_id": "0d57db99bd08c08fae9de542f4dec591cc91d0d9", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "BIBREF11 that uses a character level n-gram language model", + "'shallow' naive Bayes classifiers BIBREF12, BIBREF8, BIBREF13, BIBREF14, SVMs BIBREF15", + "BIBREF16 used an SVM with character n-gram, parts of speech tag features and some other engineered features", + "The winning approach for DSL 2015 used an ensemble naive Bayes classifier", + "The fasttext classifier BIBREF17", + "hierarchical stacked classifiers (including lexicons)", + "bidirectional recurrent neural networks BIBREF23 or ensembles of recurrent neural networks BIBREF24" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "Existing NLP datasets, models and services BIBREF10 are available for South African languages. These include an LID algorithm BIBREF11 that uses a character level n-gram language model. Multiple papers have shown that 'shallow' naive Bayes classifiers BIBREF12, BIBREF8, BIBREF13, BIBREF14, SVMs BIBREF15 and similar models work very well for doing LID. The DSL 2017 paper BIBREF1, for example, gives an overview of the solutions of all of the teams that competed on the shared task and the winning approach BIBREF16 used an SVM with character n-gram, parts of speech tag features and some other engineered features. The winning approach for DSL 2015 used an ensemble naive Bayes classifier. The fasttext classifier BIBREF17 is perhaps one of the best known efficient 'shallow' text classifiers that have been used for LID .", + "Multiple papers have proposed hierarchical stacked classifiers (including lexicons) that would for example first classify a piece of text by language group and then by exact language BIBREF18, BIBREF19, BIBREF8, BIBREF0. Some work has also been done on classifying surnames between Tshivenda, Xitsonga and Sepedi BIBREF20. Additionally, data augmentation BIBREF21 and adversarial training BIBREF22 approaches are potentially very useful to reduce the requirement for data.", + "Researchers have investigated deeper LID models like bidirectional recurrent neural networks BIBREF23 or ensembles of recurrent neural networks BIBREF24. The latter is reported to achieve 95.12% in the DSL 2015 shared task. In these models text features can include character and word n-grams as well as informative character and word-level features learnt BIBREF25 from the training data. The neural methods seem to work well in tasks where more training data is available." + ], + "highlighted_evidence": [ + "Existing NLP datasets, models and services BIBREF10 are available for South African languages. These include an LID algorithm BIBREF11 that uses a character level n-gram language model. Multiple papers have shown that 'shallow' naive Bayes classifiers BIBREF12, BIBREF8, BIBREF13, BIBREF14, SVMs BIBREF15 and similar models work very well for doing LID. The DSL 2017 paper BIBREF1, for example, gives an overview of the solutions of all of the teams that competed on the shared task and the winning approach BIBREF16 used an SVM with character n-gram, parts of speech tag features and some other engineered features. The winning approach for DSL 2015 used an ensemble naive Bayes classifier. The fasttext classifier BIBREF17 is perhaps one of the best known efficient 'shallow' text classifiers that have been used for LID .", + "Multiple papers have proposed hierarchical stacked classifiers (including lexicons) that would for example first classify a piece of text by language group and then by exact language BIBREF18, BIBREF19, BIBREF8, BIBREF0.", + "Researchers have investigated deeper LID models like bidirectional recurrent neural networks BIBREF23 or ensembles of recurrent neural networks BIBREF24. The latter is reported to achieve 95.12% in the DSL 2015 shared task." + ] + }, + "annotation_id": "9526eb3b524f649392369fbd540d603fcc9f5d88", + "worker_id": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4" + } + ] + }, + { + "question": "Is the lexicon the same for all languages?", + "question_id": "c598028815066089cc1e131b96d6966d2610467a", + "nlp_background": "two", + "topic_background": "familiar", + "paper_read": "no", + "search_query": "", + "question_writer": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": true, + "free_form_answer": "", + "evidence": [ + "The lexicon based classifier is then used to predict the specific language within a language group. For the South African languages this is done for the Nguni and Sotho groups. If the lexicon prediction of the specific language has high confidence then its result is used as the final label else the naive Bayesian classifier's specific language prediction is used as the final result. The lexicon is built over all the data and therefore includes the vocabulary from both the training and testing sets." + ], + "highlighted_evidence": [ + "The lexicon is built over all the data and therefore includes the vocabulary from both the training and testing sets." + ] + }, + "annotation_id": "ceb8d56393e4f908ac45212dd82eafc9c217d323", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": true, + "free_form_answer": "", + "evidence": [ + "The lexicon based classifier is then used to predict the specific language within a language group. For the South African languages this is done for the Nguni and Sotho groups. If the lexicon prediction of the specific language has high confidence then its result is used as the final label else the naive Bayesian classifier's specific language prediction is used as the final result. The lexicon is built over all the data and therefore includes the vocabulary from both the training and testing sets." + ], + "highlighted_evidence": [ + "The lexicon is built over all the data and therefore includes the vocabulary from both the training and testing sets." + ] + }, + "annotation_id": "f9fd7a3838b67063647c47c435953fca46c0cf6b", + "worker_id": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4" + } + ] + }, + { + "question": "How do they obtain the lexicon?", + "question_id": "ca4daafdc23f4e23d933ebabe682e1fe0d4b95ed", + "nlp_background": "two", + "topic_background": "familiar", + "paper_read": "no", + "search_query": "", + "question_writer": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4", + "answers": [ + { + "answer": { + "unanswerable": true, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "", + "evidence": [], + "highlighted_evidence": [] + }, + "annotation_id": "94076d78c3a81a9cb51b16b02340f8df58f639ef", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "built over all the data and therefore includes the vocabulary from both the training and testing sets" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "The lexicon based classifier is then used to predict the specific language within a language group. For the South African languages this is done for the Nguni and Sotho groups. If the lexicon prediction of the specific language has high confidence then its result is used as the final label else the naive Bayesian classifier's specific language prediction is used as the final result. The lexicon is built over all the data and therefore includes the vocabulary from both the training and testing sets." + ], + "highlighted_evidence": [ + "The lexicon is built over all the data and therefore includes the vocabulary from both the training and testing sets." + ] + }, + "annotation_id": "e684d010871967e599ea1de3a66ca78fefd5e90c", + "worker_id": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4" + } + ] + }, + { + "question": "What evaluation metric is used?", + "question_id": "0ab3df10f0b7203e859e9b62ffa7d6d79ffbbe50", + "nlp_background": "two", + "topic_background": "familiar", + "paper_read": "no", + "search_query": "", + "question_writer": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "average classification accuracy" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "The average classification accuracy results are summarised in Table TABREF9. The accuracies reported are for classifying a piece of text by its specific language label. Classifying text only by language group or family is a much easier task as reported in BIBREF8." + ], + "highlighted_evidence": [ + "The average classification accuracy results are summarised in Table TABREF9." + ] + }, + "annotation_id": "7a75e1c45c5b7e54f8d3bf675b477ea9c5948d5a", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "average classification accuracy", + "execution performance" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "The average classification accuracy results are summarised in Table TABREF9. The accuracies reported are for classifying a piece of text by its specific language label. Classifying text only by language group or family is a much easier task as reported in BIBREF8.", + "The execution performance of some of the LID implementations are shown in Table TABREF10. Results were generated on an early 2015 13-inch Retina MacBook Pro with a 2.9 GHz CPU (Turbo Boosted to 3.4 GHz) and 8GB RAM. The C++ implementation in BIBREF17 is the fastest. The implementation in BIBREF8 makes use of un-hashed feature representations which causes it to be slower than the proposed sklearn implementation. The execution performance of BIBREF23 might improve by a factor of five to ten when executed on a GPU." + ], + "highlighted_evidence": [ + "The average classification accuracy results are summarised in Table TABREF9. The accuracies reported are for classifying a piece of text by its specific language label.", + "The execution performance of some of the LID implementations are shown in Table TABREF10." + ] + }, + "annotation_id": "fb9f47f34736e65e1bc8fb3a4b515f02fe2ee9da", + "worker_id": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4" + } + ] + }, + { + "question": "Which languages are similar to each other?", + "question_id": "92dfacbbfa732ecea006e251be415a6f89fb4ec6", + "nlp_background": "two", + "topic_background": "familiar", + "paper_read": "no", + "search_query": "", + "question_writer": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "Nguni languages (zul, xho, nbl, ssw)", + "Sotho languages (nso, sot, tsn)" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "Table TABREF2 shows the percentages of first language speakers for each of the official languages of South Africa. These are four conjunctively written Nguni languages (zul, xho, nbl, ssw), Afrikaans (afr) and English (eng), three disjunctively written Sotho languages (nso, sot, tsn), as well as tshiVenda (ven) and Xitsonga (tso). The Nguni languages are similar to each other and harder to distinguish. The same is true of the Sotho languages." + ], + "highlighted_evidence": [ + "These are four conjunctively written Nguni languages (zul, xho, nbl, ssw), Afrikaans (afr) and English (eng), three disjunctively written Sotho languages (nso, sot, tsn), as well as tshiVenda (ven) and Xitsonga (tso). The Nguni languages are similar to each other and harder to distinguish. The same is true of the Sotho languages.", + "Similar languages are to each other are:\n- Nguni languages: zul, xho, nbl, ssw\n- Sotho languages: nso, sot, tsn" + ] + }, + "annotation_id": "30a61eebde7880653ff47562301753b54a375d75", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "The Nguni languages are similar to each other", + "The same is true of the Sotho languages" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "Table TABREF2 shows the percentages of first language speakers for each of the official languages of South Africa. These are four conjunctively written Nguni languages (zul, xho, nbl, ssw), Afrikaans (afr) and English (eng), three disjunctively written Sotho languages (nso, sot, tsn), as well as tshiVenda (ven) and Xitsonga (tso). The Nguni languages are similar to each other and harder to distinguish. The same is true of the Sotho languages." + ], + "highlighted_evidence": [ + "Table TABREF2 shows the percentages of first language speakers for each of the official languages of South Africa. These are four conjunctively written Nguni languages (zul, xho, nbl, ssw), Afrikaans (afr) and English (eng), three disjunctively written Sotho languages (nso, sot, tsn), as well as tshiVenda (ven) and Xitsonga (tso). The Nguni languages are similar to each other and harder to distinguish. The same is true of the Sotho languages." + ] + }, + "annotation_id": "36c33be6b782b952685459b871759039ec86e5c1", + "worker_id": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4" + } + ] + }, + { + "question": "Which datasets are employed for South African languages LID?", + "question_id": "c8541ff10c4e0c8e9eb37d9d7ea408d1914019a9", + "nlp_background": "two", + "topic_background": "research", + "paper_read": "no", + "search_query": "language identification", + "question_writer": "486a870694ba60f1a1e7e4ec13e328164cd4b43c", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "DSL 2015", + "DSL 2017", + "JW300 parallel corpus ", + "NCHLT text corpora" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "The focus of this section is on recently published datasets and LID research applicable to the South African context. An in depth survey of algorithms, features, datasets, shared tasks and evaluation methods may be found in BIBREF0.", + "The datasets for the DSL 2015 & DSL 2017 shared tasks BIBREF1 are often used in LID benchmarks and also available on Kaggle . The DSL datasets, like other LID datasets, consists of text sentences labelled by language. The 2017 dataset, for example, contains 14 languages over 6 language groups with 18000 training samples and 1000 testing samples per language.", + "The recently published JW300 parallel corpus BIBREF2 covers over 300 languages with around 100 thousand parallel sentences per language pair on average. In South Africa, a multilingual corpus of academic texts produced by university students with different mother tongues is being developed BIBREF3. The WiLI-2018 benchmark dataset BIBREF4 for monolingual written natural language identification includes around 1000 paragraphs of 235 languages. A possibly useful link can also be made BIBREF5 between Native Language Identification (NLI) (determining the native language of the author of a text) and Language Variety Identification (LVI) (classification of different varieties of a single language) which opens up more datasets. The Leipzig Corpora Collection BIBREF6, the Universal Declaration of Human Rights and Tatoeba are also often used sources of data.", + "The NCHLT text corpora BIBREF7 is likely a good starting point for a shared LID task dataset for the South African languages BIBREF8. The NCHLT text corpora contains enough data to have 3500 training samples and 600 testing samples of 300+ character sentences per language. Researchers have recently started applying existing algorithms for tasks like neural machine translation in earnest to such South African language datasets BIBREF9." + ], + "highlighted_evidence": [ + "The focus of this section is on recently published datasets and LID research applicable to the South African context. An in depth survey of algorithms, features, datasets, shared tasks and evaluation methods may be found in BIBREF0.\n\nThe datasets for the DSL 2015 & DSL 2017 shared tasks BIBREF1 are often used in LID benchmarks and also available on Kaggle .", + "The recently published JW300 parallel corpus BIBREF2 covers over 300 languages with around 100 thousand parallel sentences per language pair on average.", + "The WiLI-2018 benchmark dataset BIBREF4 for monolingual written natural language identification includes around 1000 paragraphs of 235 languages.", + "The NCHLT text corpora BIBREF7 is likely a good starting point for a shared LID task dataset for the South African languages BIBREF8." + ] + }, + "annotation_id": "916a3872317bb3ee04090bc633b5996a0c06f7a7", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + } + ] + }, + { + "question": "Does the paper report the performance of a baseline model on South African languages LID?", + "question_id": "307e8ab37b67202fe22aedd9a98d9d06aaa169c5", + "nlp_background": "two", + "topic_background": "research", + "paper_read": "no", + "search_query": "language identification", + "question_writer": "486a870694ba60f1a1e7e4ec13e328164cd4b43c", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": true, + "free_form_answer": "", + "evidence": [ + "FLOAT SELECTED: Table 2: LID Accuracy Results. The models we executed ourselves are marked with *. The results that are not available from our own tests or the literature are indicated with ’—’." + ], + "highlighted_evidence": [ + "FLOAT SELECTED: Table 2: LID Accuracy Results. The models we executed ourselves are marked with *. The results that are not available from our own tests or the literature are indicated with ’—’." + ] + }, + "annotation_id": "0c160bb4ef91ca885e88256f4ef1c78438757418", + "worker_id": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": true, + "free_form_answer": "", + "evidence": [ + "The average classification accuracy results are summarised in Table TABREF9. The accuracies reported are for classifying a piece of text by its specific language label. Classifying text only by language group or family is a much easier task as reported in BIBREF8.", + "FLOAT SELECTED: Table 2: LID Accuracy Results. The models we executed ourselves are marked with *. The results that are not available from our own tests or the literature are indicated with ’—’." + ], + "highlighted_evidence": [ + "The average classification accuracy results are summarised in Table TABREF9.", + "FLOAT SELECTED: Table 2: LID Accuracy Results. The models we executed ourselves are marked with *. The results that are not available from our own tests or the literature are indicated with ’—’." + ] + }, + "annotation_id": "a3e460e32adf0308aa3f317eb2a256609b2ccd46", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + } + ] + }, + { + "question": "What are the languages represented in the DSL datasets? ", + "question_id": "6415f38a06c2f99e8627e8ba6251aa4b364ade2d", + "nlp_background": "two", + "topic_background": "research", + "paper_read": "no", + "search_query": "language identification", + "question_writer": "486a870694ba60f1a1e7e4ec13e328164cd4b43c", + "answers": [ + { + "answer": { + "unanswerable": true, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "", + "evidence": [], + "highlighted_evidence": [] + }, + "annotation_id": "d6512f006a260dd1f37f861425866ab24928dec2", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + } + ] + }, + { + "question": "Does the algorithm improve on the state-of-the-art methods?", + "question_id": "e5c8e9e54e77960c8c26e8e238168a603fcdfcc6", + "nlp_background": "two", + "topic_background": "research", + "paper_read": "no", + "search_query": "language identification", + "question_writer": "486a870694ba60f1a1e7e4ec13e328164cd4b43c", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": true, + "free_form_answer": "", + "evidence": [ + "FLOAT SELECTED: Table 2: LID Accuracy Results. The models we executed ourselves are marked with *. The results that are not available from our own tests or the literature are indicated with ’—’." + ], + "highlighted_evidence": [ + "FLOAT SELECTED: Table 2: LID Accuracy Results. The models we executed ourselves are marked with *. The results that are not available from our own tests or the literature are indicated with ’—’." + ] + }, + "annotation_id": "49e1a8fc8beee0ebd41f5e02abad2468b7740d03", + "worker_id": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "From all reported results proposed method (NB+Lex) shows best accuracy on all 3 datasets - some models are not evaluated and not available in literature.", + "evidence": [ + "FLOAT SELECTED: Table 2: LID Accuracy Results. The models we executed ourselves are marked with *. The results that are not available from our own tests or the literature are indicated with ’—’.", + "The average classification accuracy results are summarised in Table TABREF9. The accuracies reported are for classifying a piece of text by its specific language label. Classifying text only by language group or family is a much easier task as reported in BIBREF8." + ], + "highlighted_evidence": [ + "FLOAT SELECTED: Table 2: LID Accuracy Results. The models we executed ourselves are marked with *. The results that are not available from our own tests or the literature are indicated with ’—’.", + "The average classification accuracy results are summarised in Table TABREF9. The accuracies reported are for classifying a piece of text by its specific language label." + ] + }, + "annotation_id": "548caee8254473df493659c215beb51e1386794a", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + } + ] + } + ], + "figures_and_tables": [ + { + "file": "2-Table1-1.png", + "caption": "Table 1: Percentage of South Africans by First Language" + }, + { + "file": "4-Table2-1.png", + "caption": "Table 2: LID Accuracy Results. The models we executed ourselves are marked with *. The results that are not available from our own tests or the literature are indicated with ’—’." + }, + { + "file": "4-Table3-1.png", + "caption": "Table 3: LID requests/sec. on NCHLT dataset. The models we executed ourselves are marked with *. For the other models the results that are not available in the literature are indicated with ’—’." + } + ] + }, + "1707.06806": { + "title": "Shallow reading with Deep Learning: Predicting popularity of online content using only its title", + "abstract": "With the ever decreasing attention span of contemporary Internet users, the title of online content (such as a news article or video) can be a major factor in determining its popularity. To take advantage of this phenomenon, we propose a new method based on a bidirectional Long Short-Term Memory (LSTM) neural network designed to predict the popularity of online content using only its title. We evaluate the proposed architecture on two distinct datasets of news articles and news videos distributed in social media that contain over 40,000 samples in total. On those datasets, our approach improves the performance over traditional shallow approaches by a margin of 15%. Additionally, we show that using pre-trained word vectors in the embedding layer improves the results of LSTM models, especially when the training set is small. To our knowledge, this is the first attempt of applying popularity prediction using only textual information from the title.", + "full_text": [ + { + "section_name": "Introduction", + "paragraphs": [ + "The distribution of textual content is typically very fast and catches user attention for only a short period of time BIBREF0 . For this reason, proper wording of the article title may play a significant role in determining the future popularity of the article. The reflection of this phenomenon is the proliferation of click-baits - short snippets of text whose main purpose is to encourage viewers to click on the link embedded in the snippet. Although detection of click-baits is a separate research topic BIBREF1 , in this paper we address a more general problem of predicting popularity of online content based solely on its title.", + "Predicting popularity in the Internet is a challenging and non-trivial task due to a multitude of factors impacting the distribution of the information: external context, social network of the publishing party, relevance of the video to the final user, etc. This topic has therefore attracted a lot of attention from the research community BIBREF2 , BIBREF3 , BIBREF0 , BIBREF4 .", + "In this paper we propose a method for online content popularity prediction based on a bidirectional recurrent neural network called BiLSTM. This work is inspired by recent successful applications of deep neural networks in many natural language processing problems BIBREF5 , BIBREF6 . Our method attempts to model complex relationships between the title of an article and its popularity using novel deep network architecture that, in contrast to the previous approaches, gives highly interpretable results. Last but not least, the proposed BiLSTM method provides a significant performance boost in terms of prediction accuracy over the standard shallow approach, while outperforming the current state-of-the-art on two distinct datasets with over 40,000 samples.", + "To summarize, the contributions presented in this paper are the following:", + "The remainder of this paper is organized in the following manner: first, we review the relevant literature and compare our approach to existing work. Next, we formulate the problem of popularity prediction and propose a model that takes advantage of BiLSTM architecture to address it. Then, we evaluate our model on two datasets using several pre-trained word embeddings and compare it to benchmark models. We conclude this work with discussion on future research paths." + ] + }, + { + "section_name": "Related Work", + "paragraphs": [ + "The ever increasing popularity of the Internet as a virtual space to share content inspired research community to analyze different aspects of online information distribution. Various types of content were analyzed, ranging from textual data, such as Twitter posts BIBREF0 or Digg stories BIBREF2 to images BIBREF7 to videos BIBREF8 , BIBREF3 , BIBREF9 . Although several similarities were observed across content domains, e.g. log-normal distribution of data popularity BIBREF10 , in this work we focus only on textual content and, more precisely, on the popularity of news articles and its relation to the article's title.", + "Forecasting popularity of news articles was especially well studied in the context of Twitter - a social media platform designed specifically for sharing textual data BIBREF11 , BIBREF12 . Not only did the previous works focus on the prediction part, but also on modeling message propagation within the network BIBREF13 . However, most of the works were focused on analyzing the social interactions between the users and the characteristics of so-called social graph of users' connections, rather than on the textual features. Contrary to those approaches, in this paper we base our predictions using only textual features of the article title. We also validate our proposed method on one dataset collected using a different social media platform, namely Facebook, and another one created from various news articles BIBREF4 .", + "Recently, several works have touched on the topic of popularity prediction of news article from a multimodal perspective BIBREF4 , BIBREF14 . Although in BIBREF4 the authors analyze news articles on a per-modality basis, they do not approach the problem of popularity prediction in a holistic way. To address this shortcoming, BIBREF14 have proposed a multimodal approach to predicting popularity of short videos shares in social media platform Vine using a model that fuses features related to different modalities. In our work, we focus only on textual features of the article title for the purpose of popularity prediction, as our goal is to empower the journalists to quantitatively assess the quality of the headlines they create before the publication. Nevertheless, we believe that in future research we will extend our method towards multimodal popularity prediction." + ] + }, + { + "section_name": "Method", + "paragraphs": [ + "In this section we present the bidirectional LSTM model for popularity prediction. We start by formulating the problem and follow up with the description of word embeddings used in our approach. We then present the Long Short-Term Memory network that serves as a backbone for our bidirectional LSTM architecture. We conclude this section with our interpretation of hidden bidirectional states and describe how they can be employed for title introspection." + ] + }, + { + "section_name": "Problem Formulation", + "paragraphs": [ + "We cast the problem of popularity prediction as a binary classification task. We assume our data points contain a string of characters representing article title and a popularity metric, such as number of comments or views. The input of our classification is the character string, while the output is the binary label corresponding to popular or unpopular class. To enable the comparison of the methods on datasets containing content published on different websites and with different audience sizes, we determine that a video is popular if its popularity metric exceeds the median value of the corresponding metric for other points in the set, otherwise - it is labeled as unpopular. The details of the labeling procedure are discussed separately in the Datasets section." + ] + }, + { + "section_name": "Text Representation", + "paragraphs": [ + "Since the input of our method is textual data, we follow the approach of BIBREF15 and map the text into a fixed-size vector representation. To this end, we use word embeddings that were successfully applied in other domains. We follow BIBREF5 and use pre-trained GloVe word vectors BIBREF16 to initialize the embedding layer (also known as look-up table). Section SECREF18 discusses the embedding layer in more details." + ] + }, + { + "section_name": "Bidirectional Long Short-Term Memory Network", + "paragraphs": [ + "Our method for popularity prediction using article's title is inspired by a bidirectional LSTM architecture. The overview of the model can be seen in Fig. FIGREF8 .", + "Let INLINEFORM0 be INLINEFORM1 -dimensional word vector corresponding to the INLINEFORM2 -the word in the headline, then a variable length sequence: INLINEFORM3 represents a headline. A recurrent neural network (RNN) processes this sequence by recursively applying a transformation function to the current element of sequence INLINEFORM4 and its previous hidden internal state INLINEFORM5 (optionally outputting INLINEFORM6 ). At each time step INLINEFORM7 , the hidden state is updated by: DISPLAYFORM0 ", + "where INLINEFORM0 is a non-linear activation function. LSTM network BIBREF17 updates its internal state differently, at each step INLINEFORM1 it calculates: DISPLAYFORM0 ", + " where INLINEFORM0 is the sigmoid activation function, tanh is the hyperbolic tangent function and INLINEFORM1 denotes component-wise multiplication. In our experiments we used 128, 256 for the dimensionality of hidden layer in both LSTM and BiLSTM. The term in equation EQREF10 INLINEFORM2 , is called the input gate and it uses the input word and the past hidden state to determine whether the input is worth remembering or not. The amount of information that is being discarded is controlled by forget gate INLINEFORM3 , while INLINEFORM4 is the output gate that controls the amount of information that leaks from memory cell INLINEFORM5 to the hidden state INLINEFORM6 . In the context of classification, we typically treat the output of the hidden state at the last time step of LSTM as the document representation and feed it to sigmoid layer to perform classification BIBREF18 .", + "Due to its sequential nature, a recurrent neural network puts more emphasis on the recent elements. To circumvent this problem BIBREF19 introduced a bidirectional RNN in which each training sequence is presented forwards and backwards to two separate recurrent nets, both of which are connected to the same output layer. Therefore, at any time-step we have the whole information about the sequence. This is shown by the following equation: DISPLAYFORM0 ", + "In our method, we use the bidirectional LSTM architecture for content popularity prediction using only textual cues. We have to therefore map the neural network outputs from a set of hidden states INLINEFORM0 to classification labels. We evaluated several approaches to this problem, such as max or mean pooling. The initial experiments showed that the highest performance was achieved using late fusion approach, that is by concatenating the last hidden state in forward and backward sequence. The intuition behind this design choice is that the importance of the first few words of the headline is relatively high, as the information contained in INLINEFORM1 , i.e. the last item in the backward sequence, is mostly taken from the first word." + ] + }, + { + "section_name": "Hidden State Interpretation", + "paragraphs": [ + "One interesting property of bidirectional RNNs is the fact, that the concatenation of hidden states INLINEFORM0 and INLINEFORM1 can be interpreted as a context-dependent vector representation of word INLINEFORM2 . This allows us to introspect a given title and approximate the contribution of each word to the estimated popularity. To that end one can process the headline representation INLINEFORM3 through the bidirectional recurrent network and then retrieve pairs of forward and backwards hidden state INLINEFORM4 for each word INLINEFORM5 . Then, the output of the last fully-connected layer INLINEFORM6 could be interpreted as context-depended popularity of a word INLINEFORM7 ." + ] + }, + { + "section_name": "Training", + "paragraphs": [ + "In our experiments we minimize the binary cross-entropy loss using Stochastic Gradient Descent on randomly shuffled mini-batches with the Adam optimization algorithm BIBREF20 . We reduce the learning rate by a factor of 0.2 once learning plateaus. We also employ early stopping strategy, i.e. stopping the training algorithm before convergence based on the values of loss function on the validation set." + ] + }, + { + "section_name": "Evaluation", + "paragraphs": [ + "In this section, we evaluate our method and compare its performance against the competitive approaches. We use INLINEFORM0 -fold evaluation protocol with INLINEFORM1 with random dataset split. We measure the performance using standard accuracy metric which we define as a ratio between correctly classified data samples from test dataset and all test samples." + ] + }, + { + "section_name": "Datasets", + "paragraphs": [ + "In this section we present two datasets used in our experiments: The NowThisNews dataset, collected for the purpose of this paper, and The BreakingNews dataset BIBREF4 , publicly available dataset of news articles.", + "contains 4090 posts with associated videos from NowThisNews Facebook page collected between 07/2015 and 07/2016. For each post we collected its title and the number of views of the corresponding video, which we consider our popularity metric. Due to a fairly lengthy data collection process, we decided to normalize our data by first grouping posts according to their publication month and then labeling the posts for which the popularity metric exceeds the median monthly value as popular, the remaining part as unpopular.", + " BIBREF4 contains a variety of news-related information such as images, captions, geo-location information and comments which could be used as a proxy for article popularity. The articles in this dataset were collected between January and December 2014. Although we tried to retrieve the entire dataset, we were able to download only 38,182 articles due to the dead links published in the dataset. The retrieved articles were published in main news channels, such as Yahoo News, The Guardian or The Washington Post. Similarly, to The NowThisNews dataset we normalize the data by grouping articles per publisher, and classifying them as popular, when the number of comments exceeds the median value for given publisher." + ] + }, + { + "section_name": "Baselines", + "paragraphs": [ + "As a first baseline we use Bag-of-Words, a well-known and robust text representations used in various domains BIBREF21 , combined with a standard shallow classifier, namely, a Support Vector Machine with linear kernel. We used LIBSVM implementation of SVM.", + "Our second baseline is a deep Convectional Neural Network applied on word embeddings. This baseline represents state-of-the-art method presented in BIBREF4 with minor adjustments to the binary classification task. The architecture of the CNN benchmark we use is the following: the embedding layer transforms one-hot encoded words to their dense vector representations, followed by the convolution layer of 256 filters with width equal to 5 followed by max pooling layer (repeated three times), fully-connected layer with dropout and INLINEFORM0 regularization and finally, sigmoid activation layer. For fair comparison, both baselines were trained using the same training procedure as our method." + ] + }, + { + "section_name": "Embeddings", + "paragraphs": [ + "As a text embedding in our experiments, we use publicly available GloVe word vectors BIBREF16 pre-trained on two datasets: Wikipedia 2014 with Gigaword5 (W+G5) and Common Crawl (CC). Since their output dimensionality can be modified, we show the results for varying dimensionality sizes. On top of that, we evaluate two training approaches: using static word vectors and fine-tuning them during training phase." + ] + }, + { + "section_name": "Results", + "paragraphs": [ + "The results of our experiments can be seen in Tab. TABREF21 and TABREF22 . Our proposed BiLSTM approach outperforms the competing methods consistently across both datasets. The performance improvement is especially visible for The NowThisNews dataset and reaches over 15% with respect to the shallow architecture in terms of of accuracy. Although the improvement with respect to the other methods based on deep neural network is less evident, the recurrent nature of our method provides much more intuitive interpretation of the results and allow for parsing the contribution of each single word to the overall score.", + "To present how our model works in practice, we show in Tab. TABREF23 a list of 3 headlines from NowThisNews dataset that are scored with the highest probability of belonging to a popular class, as well as 3 headlines with the lowest score. As can be seen, our model correctly detected videos that become viral at the same time assigning low score to content that underperformed. We believe that BiLSTM could be successfully applied in real-life scenarios." + ] + }, + { + "section_name": "Conclusions", + "paragraphs": [ + "In this paper we present a novel approach to the problem of online article popularity prediction. To our knowledge, this is the first attempt of predicting the performance of content on social media using only textual information from its title. We show that our method consistently outperforms benchmark models. Additionally, the proposed method could not only be used to compare competing titles with regard to their estimated probability, but also to gain insights about what constitutes a good title. Future work includes modeling popularity prediction problem with multiple data modalities, such as images or videos. Furthermore, all of the evaluated models function at the word level, which could be problematic due to idiosyncratic nature of social media and Internet content. It is, therefore, worth investigating, whether combining models that operate at the character level to learn and generate vector representation of titles with visual features could improve the overall performance." + ] + } + ], + "qas": [ + { + "question": "What is the average length of the title text?", + "question_id": "252a645af9876241fb166e5822992ce17fec6eb6", + "nlp_background": "", + "topic_background": "", + "paper_read": "", + "search_query": "", + "question_writer": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4", + "answers": [ + { + "answer": { + "unanswerable": true, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "", + "evidence": [], + "highlighted_evidence": [] + }, + "annotation_id": "4073d618697ed6e9ac55649d7eaadcc80ae443e5", + "worker_id": "71f73551e7aabf873649e8fe97aefc54e6dd14f8" + } + ] + }, + { + "question": "Which pretrained word vectors did they use?", + "question_id": "ed67359889cf61fa11ee291d6c378cccf83d599d", + "nlp_background": "", + "topic_background": "", + "paper_read": "", + "search_query": "", + "question_writer": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + " pre-trained GloVe word vectors " + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "Since the input of our method is textual data, we follow the approach of BIBREF15 and map the text into a fixed-size vector representation. To this end, we use word embeddings that were successfully applied in other domains. We follow BIBREF5 and use pre-trained GloVe word vectors BIBREF16 to initialize the embedding layer (also known as look-up table). Section SECREF18 discusses the embedding layer in more details." + ], + "highlighted_evidence": [ + "Since the input of our method is textual data, we follow the approach of BIBREF15 and map the text into a fixed-size vector representation. To this end, we use word embeddings that were successfully applied in other domains. We follow BIBREF5 and use pre-trained GloVe word vectors BIBREF16 to initialize the embedding layer (also known as look-up table). Section SECREF18 discusses the embedding layer in more details." + ] + }, + "annotation_id": "60a07a216a2ad8dca51926d57bdf73c2c9843e6f", + "worker_id": "a0b403873302db7cada39008f04d01155ef68f4f" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "GloVe word vectors BIBREF16 pre-trained on two datasets: Wikipedia 2014 with Gigaword5 (W+G5) and Common Crawl (CC)" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "As a text embedding in our experiments, we use publicly available GloVe word vectors BIBREF16 pre-trained on two datasets: Wikipedia 2014 with Gigaword5 (W+G5) and Common Crawl (CC). Since their output dimensionality can be modified, we show the results for varying dimensionality sizes. On top of that, we evaluate two training approaches: using static word vectors and fine-tuning them during training phase." + ], + "highlighted_evidence": [ + "As a text embedding in our experiments, we use publicly available GloVe word vectors BIBREF16 pre-trained on two datasets: Wikipedia 2014 with Gigaword5 (W+G5) and Common Crawl (CC). " + ] + }, + "annotation_id": "66146081f26b64f467e66e62e791f890fbbc0560", + "worker_id": "3c0d42931aaae53acefbee56b67ca230244422b4" + } + ] + }, + { + "question": "What evaluation metrics are used?", + "question_id": "425bd2ccfd95ead91d8f2b1b1c8ab9fc3446cb82", + "nlp_background": "", + "topic_background": "", + "paper_read": "", + "search_query": "", + "question_writer": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "standard accuracy metric" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "In this section, we evaluate our method and compare its performance against the competitive approaches. We use INLINEFORM0 -fold evaluation protocol with INLINEFORM1 with random dataset split. We measure the performance using standard accuracy metric which we define as a ratio between correctly classified data samples from test dataset and all test samples." + ], + "highlighted_evidence": [ + "In this section, we evaluate our method and compare its performance against the competitive approaches. We use INLINEFORM0 -fold evaluation protocol with INLINEFORM1 with random dataset split. We measure the performance using standard accuracy metric which we define as a ratio between correctly classified data samples from test dataset and all test samples." + ] + }, + "annotation_id": "7d4d3a5e45f844f8ec83e4d59f83ed20db1926be", + "worker_id": "a0b403873302db7cada39008f04d01155ef68f4f" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "accuracy" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "In this paper we propose a method for online content popularity prediction based on a bidirectional recurrent neural network called BiLSTM. This work is inspired by recent successful applications of deep neural networks in many natural language processing problems BIBREF5 , BIBREF6 . Our method attempts to model complex relationships between the title of an article and its popularity using novel deep network architecture that, in contrast to the previous approaches, gives highly interpretable results. Last but not least, the proposed BiLSTM method provides a significant performance boost in terms of prediction accuracy over the standard shallow approach, while outperforming the current state-of-the-art on two distinct datasets with over 40,000 samples." + ], + "highlighted_evidence": [ + "Last but not least, the proposed BiLSTM method provides a significant performance boost in terms of prediction accuracy over the standard shallow approach, while outperforming the current state-of-the-art on two distinct datasets with over 40,000 samples." + ] + }, + "annotation_id": "866675dc6dfbe9c44dcd371db052457a6b7d47ca", + "worker_id": "71f73551e7aabf873649e8fe97aefc54e6dd14f8" + } + ] + }, + { + "question": "Which shallow approaches did they experiment with?", + "question_id": "955de9f7412ba98a0c91998919fa048d339b1d48", + "nlp_background": "", + "topic_background": "", + "paper_read": "", + "search_query": "", + "question_writer": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "SVM" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "As a first baseline we use Bag-of-Words, a well-known and robust text representations used in various domains BIBREF21 , combined with a standard shallow classifier, namely, a Support Vector Machine with linear kernel. We used LIBSVM implementation of SVM." + ], + "highlighted_evidence": [ + "As a first baseline we use Bag-of-Words, a well-known and robust text representations used in various domains BIBREF21 , combined with a standard shallow classifier, namely, a Support Vector Machine with linear kernel. We used LIBSVM implementation of SVM." + ] + }, + "annotation_id": "69ba89d3c1980a1afa93a066afb621eb612d8d14", + "worker_id": "a0b403873302db7cada39008f04d01155ef68f4f" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "SVM with linear kernel using bag-of-words features", + "evidence": [ + "As a first baseline we use Bag-of-Words, a well-known and robust text representations used in various domains BIBREF21 , combined with a standard shallow classifier, namely, a Support Vector Machine with linear kernel. We used LIBSVM implementation of SVM." + ], + "highlighted_evidence": [ + "As a first baseline we use Bag-of-Words, a well-known and robust text representations used in various domains BIBREF21 , combined with a standard shallow classifier, namely, a Support Vector Machine with linear kernel." + ] + }, + "annotation_id": "fa9eeba099c0d3b7b0452630d106ff71097a0010", + "worker_id": "3c0d42931aaae53acefbee56b67ca230244422b4" + } + ] + }, + { + "question": "Where do they obtain the news videos from?", + "question_id": "3b371ea554fa6639c76a364060258454e4b931d4", + "nlp_background": "", + "topic_background": "", + "paper_read": "", + "search_query": "", + "question_writer": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "NowThisNews Facebook page" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "In this section we present two datasets used in our experiments: The NowThisNews dataset, collected for the purpose of this paper, and The BreakingNews dataset BIBREF4 , publicly available dataset of news articles.", + "contains 4090 posts with associated videos from NowThisNews Facebook page collected between 07/2015 and 07/2016. For each post we collected its title and the number of views of the corresponding video, which we consider our popularity metric. Due to a fairly lengthy data collection process, we decided to normalize our data by first grouping posts according to their publication month and then labeling the posts for which the popularity metric exceeds the median monthly value as popular, the remaining part as unpopular." + ], + "highlighted_evidence": [ + "In this section we present two datasets used in our experiments: The NowThisNews dataset, collected for the purpose of this paper, and The BreakingNews dataset BIBREF4 , publicly available dataset of news articles.\n\ncontains 4090 posts with associated videos from NowThisNews Facebook page collected between 07/2015 and 07/2016. For each post we collected its title and the number of views of the corresponding video, which we consider our popularity metric. Due to a fairly lengthy data collection process, we decided to normalize our data by first grouping posts according to their publication month and then labeling the posts for which the popularity metric exceeds the median monthly value as popular, the remaining part as unpopular." + ] + }, + "annotation_id": "79ce57787f236232adaba5931ed910db7a2a505d", + "worker_id": "a0b403873302db7cada39008f04d01155ef68f4f" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "NowThisNews Facebook page" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "In this section we present two datasets used in our experiments: The NowThisNews dataset, collected for the purpose of this paper, and The BreakingNews dataset BIBREF4 , publicly available dataset of news articles.", + "contains 4090 posts with associated videos from NowThisNews Facebook page collected between 07/2015 and 07/2016. For each post we collected its title and the number of views of the corresponding video, which we consider our popularity metric. Due to a fairly lengthy data collection process, we decided to normalize our data by first grouping posts according to their publication month and then labeling the posts for which the popularity metric exceeds the median monthly value as popular, the remaining part as unpopular." + ], + "highlighted_evidence": [ + "In this section we present two datasets used in our experiments: The NowThisNews dataset, collected for the purpose of this paper, and The BreakingNews dataset BIBREF4 , publicly available dataset of news articles.", + "contains 4090 posts with associated videos from NowThisNews Facebook page collected between 07/2015 and 07/2016." + ] + }, + "annotation_id": "ef822e6efb769abafe0423614b15850e986e9db1", + "worker_id": "71f73551e7aabf873649e8fe97aefc54e6dd14f8" + } + ] + }, + { + "question": "What is the source of the news articles?", + "question_id": "ddb23a71113cbc092cbc158066d891cae261e2c6", + "nlp_background": "", + "topic_background": "", + "paper_read": "", + "search_query": "", + "question_writer": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "main news channels, such as Yahoo News, The Guardian or The Washington Post" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "BIBREF4 contains a variety of news-related information such as images, captions, geo-location information and comments which could be used as a proxy for article popularity. The articles in this dataset were collected between January and December 2014. Although we tried to retrieve the entire dataset, we were able to download only 38,182 articles due to the dead links published in the dataset. The retrieved articles were published in main news channels, such as Yahoo News, The Guardian or The Washington Post. Similarly, to The NowThisNews dataset we normalize the data by grouping articles per publisher, and classifying them as popular, when the number of comments exceeds the median value for given publisher." + ], + "highlighted_evidence": [ + "BIBREF4 contains a variety of news-related information such as images, captions, geo-location information and comments which could be used as a proxy for article popularity. The articles in this dataset were collected between January and December 2014. Although we tried to retrieve the entire dataset, we were able to download only 38,182 articles due to the dead links published in the dataset. The retrieved articles were published in main news channels, such as Yahoo News, The Guardian or The Washington Post. Similarly, to The NowThisNews dataset we normalize the data by grouping articles per publisher, and classifying them as popular, when the number of comments exceeds the median value for given publisher." + ] + }, + "annotation_id": "10562160d8bcaf723be9b384ef94e9bb8f32d493", + "worker_id": "a0b403873302db7cada39008f04d01155ef68f4f" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "The BreakingNews dataset" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "In this section we present two datasets used in our experiments: The NowThisNews dataset, collected for the purpose of this paper, and The BreakingNews dataset BIBREF4 , publicly available dataset of news articles." + ], + "highlighted_evidence": [ + "In this section we present two datasets used in our experiments: The NowThisNews dataset, collected for the purpose of this paper, and The BreakingNews dataset BIBREF4 , publicly available dataset of news articles." + ] + }, + "annotation_id": "e3987bfaef7c4dc5a6dfd1af11ad6c9d22d01fa4", + "worker_id": "71f73551e7aabf873649e8fe97aefc54e6dd14f8" + } + ] + } + ], + "figures_and_tables": [ + { + "file": "4-Figure1-1.png", + "caption": "Fig. 1. A bidirectional LSTM architecture with 1-of-K word encoding and embedding layer proposed in this paper." + }, + { + "file": "8-Table1-1.png", + "caption": "Table 1. Popularity prediction results on NowThisNews dataset. Our proposed BiLSTM method provides higher performances than the competitors in terms of classification accuracy." + }, + { + "file": "8-Table2-1.png", + "caption": "Table 2. Popularity prediction results on BreakingNews dataset. Our BiLSTM method outperforms the competitors - the performance gain is especially visible with respect to the shallow architecture of BoW + SVM." + }, + { + "file": "9-Table3-1.png", + "caption": "Table 3. Top and bottom 3 headlines from the NowThisNews dataset as predicted by our model and their views 168 hours after publication." + } + ] + }, + "1806.04511": { + "title": "Multilingual Sentiment Analysis: An RNN-Based Framework for Limited Data", + "abstract": "Sentiment analysis is a widely studied NLP task where the goal is to determine opinions, emotions, and evaluations of users towards a product, an entity or a service that they are reviewing. One of the biggest challenges for sentiment analysis is that it is highly language dependent. Word embeddings, sentiment lexicons, and even annotated data are language specific. Further, optimizing models for each language is very time consuming and labor intensive especially for recurrent neural network models. From a resource perspective, it is very challenging to collect data for different languages. In this paper, we look for an answer to the following research question: can a sentiment analysis model trained on a language be reused for sentiment analysis in other languages, Russian, Spanish, Turkish, and Dutch, where the data is more limited? Our goal is to build a single model in the language with the largest dataset available for the task, and reuse it for languages that have limited resources. For this purpose, we train a sentiment analysis model using recurrent neural networks with reviews in English. We then translate reviews in other languages and reuse this model to evaluate the sentiments. Experimental results show that our robust approach of single model trained on English reviews statistically significantly outperforms the baselines in several different languages.", + "full_text": [ + { + "section_name": "Introduction", + "paragraphs": [ + "With the steady growth in the commercial websites and social media venues, the access to users' reviews have become easier. As the amount of data that can be mined for opinion increased, commercial companies' interests for sentiment analysis increased as well. Sentiment analysis is an important part of understanding user behavior and opinions on products, places, or services.", + "Sentiment analysis has long been studied by the research community, leading to several sentiment-related resources such as sentiment dictionaries that can be used as features for machine learning models BIBREF0 , BIBREF1 , BIBREF2 , BIBREF3 . These resources help increase sentiment analysis accuracies; however, they are highly dependent on language and require researchers to build such resources for every language to process.", + "Feature engineering is a large part of the model building phase for most sentiment analysis and emotion detection models BIBREF4 . Determining the correct set of features is a task that requires thorough investigation. Furthermore, these features are mostly language and dataset dependent making it even further challenging to build models for different languages. For example, the sentiment and emotion lexicons, as well as pre-trained word embeddings are not completely transferable to other languages which replicates the efforts for every language that users would like to build sentiment classification models on. For languages and tasks where the data is limited, extracting these features, building language models, training word embeddings, and creating lexicons are big challenges. In addition to the feature engineering effort, the machine learning models' parameters also need to be tuned separately for each language to get the optimal results.", + "In this paper, we take a different approach. We build a reusable sentiment analysis model that does not utilize any lexicons. Our goal is to evaluate how well a generic model can be used to mine opinion in different languages where data is more limited than the language where the generic model is trained on. To that end, we build a training set that contains reviews from different domains in English (e.g., movie reviews, product reviews) and train a recurrent neural network (RNN) model to predict polarity of those reviews. Then focusing on a domain, we make the model specialized in that domain by using the trained weights from the larger data and further training with data on a specific domain. To evaluate the reusability of the sentiment analysis model, we test with non-English datasets. We first translate the test set to English and use the pre-trained model to score polarity in the translated text. In this way, our proposed approach eliminates the need to train language-dependent models, use of sentiment lexicons and word embeddings for each language. Our experiments show that a generalizable sentiment analysis model can be utilized successfully to perform opinion mining for languages that do not have enough resources to train specific models.", + "The contributions of this study are; 1) a robust approach that utilizes machine translation to reuse a model trained on one language in other languages, 2) an RNN-based approach to eliminate feature extraction as well as resource requirements for sentiment analysis, and 3) a technique that statistically significantly outperforms baselines for multilingual sentiment analysis task when data is limited. To the best of our knowledge, this study is the first to apply a deep learning model to the multilingual sentiment analysis task." + ] + }, + { + "section_name": "Related Work", + "paragraphs": [ + "There is a rich body of work in sentiment analysis including social media platforms such as Twitter BIBREF5 and Facebook BIBREF4 . One common factor in most of the sentiment analysis work is that features that are specific to sentiment analysis are extracted (e.g., sentiment lexicons) and used in different machine learning models. Lexical resources BIBREF0 , BIBREF1 , BIBREF4 for sentiment analysis such as SentiWordNet BIBREF6 , BIBREF7 , linguistic features and expressions BIBREF8 , polarity dictionaries BIBREF2 , BIBREF3 , other features such as topic-oriented features and syntax BIBREF9 , emotion tokens BIBREF10 , word vectors BIBREF11 , and emographics BIBREF12 are some of the information that are found useful for improving sentiment analysis accuracies. Although these features are beneficial, extracting them requires language-dependent data (e.g., a sentiment dictionary for Spanish is trained on Spanish data instead of using all data from different languages).", + "Our goal in this work is to streamline the feature engineering phase by not relying on any dictionary other than English word embeddings that are trained on any data (i.e. not necessarily sentiment analysis corpus). To that end, we utilize off-the-shelf machine translation tools to first translate corpora to the language where more training data is available and use the translated corpora to do inference on.", + "Machine translation for multilingual sentiment analysis has also seen attention from researchers. Hiroshi et al. BIBREF13 translated only sentiment units with a pattern-based approach. Balahur and Turchi BIBREF14 used uni-grams, bi-grams and tf-idf features for building support vector machines on translated text. Boyd-Graber and Resnik BIBREF15 built Latent Dirichlet Allocation models to investigate how multilingual concepts are clustered into topics. Mohammed et al. BIBREF16 translate Twitter posts to English as well as the English sentiment lexicons. Tellez et al. BIBREF17 propose a framework where language-dependent and independent features are used with an SVM classifier. These machine learning approaches also require a feature extraction phase where we eliminate by incorporating a deep learning approach that does the feature learning intrinsically. Further, Wan BIBREF18 uses an ensemble approach where the resources (e.g., lexicons) in both the original language and the translated language are used – requiring resources to be present in both languages. Brooke et al. BIBREF19 also use multiple dictionaries.", + "In this paper, we address the resource bottleneck of these translation-based approaches and propose a deep learning approach that does not require any dictionaries." + ] + }, + { + "section_name": "Methodology", + "paragraphs": [ + "In order to eliminate the need to find data and build separate models for each language, we propose a multilingual approach where a single model is built in the language where the largest resources are available. In this paper we focus on English as there are several sentiment analysis datasets in English. To make the English sentiment analysis model as generalizable as possible, we first start by training with a large dataset that has product reviews for different categories. Then, using the trained weights from the larger generic dataset, we make the model more specialized for a specific domain. We further train the model with domain-specific English reviews and use this trained model to score reviews that share the same domain from different languages. To be able to employ the trained model, test sets are first translated to English via machine translation and then inference takes place. Figure FIGREF1 shows our multilingual sentiment analysis approach. It is important to note that this approach does not utilize any resource in any of the languages of the test sets (e.g., word embeddings, lexicons, training set).", + "Deep learning approaches have been successful in many applications ranging from computer vision to natural language processing BIBREF20 . Recurrent neural network (RNN) including Long Short Term Memory (LSTM) and Gated Recurrent Units (GRU) are subsets of deep learning algorithms where the dependencies between tokens can be used by the model. These models can also be used with variable length input vectors which makes them suitable for text input. LSTM and GRU models allow operations of sequences of vectors over time and have the capability to `remember' previous information BIBREF20 . RNN have been found useful for several natural language processing tasks including language modeling, text classification, machine translation. RNN can also utilize pre-trained word embeddings (numeric vector representations of words trained on unlabeled data) without requiring hand-crafted features. Therefore in this paper, we employ an RNN architecture that takes text and pre-trained word embeddings as inputs and generates a classification result. Word embeddings represent words as numeric vectors and capture semantic information. They are trained in an unsupervised fashion making it useful for our task.", + "The sentiment analysis model that is trained on English reviews has two bidirectional layers, each with 40 neurons and a dropout BIBREF21 of 0.2 is used. The training phase takes pre-trained word embeddings and reviews in textual format, then predicts the polarity of the reviews. For this study, an embedding length of 100 is used (i.e., each word is represented by a vector of length 100). We utilized pre-trained global vectors BIBREF22 . The training phase is depicted in Figure FIGREF2 ." + ] + }, + { + "section_name": "Experiments", + "paragraphs": [ + "To evaluate the proposed approach for multilingual sentiment analysis task, we conducted experiments. This section first presents the corpora used in this study followed by experimental results.", + "Throughout our experiments, we use SAS Deep Learning Toolkit. For machine translation, Google translation API is used." + ] + }, + { + "section_name": "Corpora", + "paragraphs": [ + "Two sets of corpora are used in this study, both are publicly available. The first set consists of English reviews and the second set contains restaurant reviews from four different languages (Spanish, Turkish, Dutch, Russian). We focus on polarity detection in reviews, therefore all datasets in this study have two class values (positive, negative).", + "With the goal of building a generalizable sentiment analysis model, we used three different training sets as provided in Table TABREF5 . One of these three datasets (Amazon reviews BIBREF23 , BIBREF24 ) is larger and has product reviews from several different categories including book reviews, electronics products reviews, and application reviews. The other two datasets are to make the model more specialized in the domain. In this paper we focus on restaurant reviews as our domain and use Yelp restaurant reviews dataset extracted from Yelp Dataset Challenge BIBREF25 and restaurant reviews dataset as part of a Kaggle competition BIBREF26 .", + "For evaluation of the multilingual approach, we use four languages. These datasets are part of SemEval-2016 Challenge Task 5 BIBREF27 , BIBREF28 . Table TABREF7 shows the number of observations in each test corpus." + ] + }, + { + "section_name": "Experimental Results", + "paragraphs": [ + "For experimental results, we report majority baseline for each language where the majority baseline corresponds to a model's accuracy if it always predicts the majority class in the dataset. For example, if the dataset has 60% of all reviews positive and 40% negative, majority baseline would be 60% because a model that always predicts “positive” will be 60% accurate and will make mistakes 40% of the time.", + "In addition to the majority baseline, we also compare our results with a lexicon-based approach. We use SentiWordNet BIBREF29 to obtain a positive and a negative sentiment score for each token in a review. Then sum of positive sentiment scores and negative sentiment scores for each review is obtained by summing up the scores for each token. If the positive sum score for a given review is greater than the negative sum score, we accept that review as a positive review. If negative sum is larger than or equal to the positive sum, the review is labeled as a negative review.", + "RNN outperforms both baselines in all four datasets (see Table TABREF9 ). Also for Spanish restaurant review, the lexicon-based baseline is below the majority baseline which shows that solely translating data and using lexicons is not sufficient to achieve good results in multilingual sentiment analysis.", + "Among the wrong classifications for each test set, we calculated the percentage of false positives and false negatives. Table TABREF10 shows the distribution of false positives and false negatives for each class. In all four classes, the number of false negatives are more than the number of false positives. This can be explained by the unbalanced training dataset where the number of positive reviews are more than the number of negative reviews (59,577 vs 17,132).", + "To be able to see the difference between baseline and RNN, we took each method's results as a group (4 values: one for each language) and compared the means. Post hoc comparisons using the Tukey HSD test indicated that the mean accuracies for baselines (majority and lexicon-based) are significantly different than RNN accuracies as can be seen in Table TABREF12 (family-wise error rate=0.06). When RNN is compared with lexicon-based baseline and majority baseline, the null hypothesis can be rejected meaning that each test is significant. In addition to these comparisons, we also calculated the effect sizes (using Cohen's d) between the baselines and our method. The results are aligning with Tukey HSD results such that while our method versus baselines have very large effect sizes, lexicon-based baseline and majority baseline have negligible effect size.", + "Figure FIGREF11 shows the differences in minimum and maximum values of all three approaches. As the figure shows, RNN significantly outperforms both baselines for the sentiment classification task." + ] + }, + { + "section_name": "Discussion", + "paragraphs": [ + "One of the crucial elements while using machine translation is to have highly accurate translations. It is likely that non-English words would not have word embeddings, which will dramatically affect the effectiveness of the system. We analyzed the effect of incorrect translations into our approach. To that end, we extracted all wrong predictions from the test set and computed the ratio of misclassifications that have non-English words in them. We first extracted all misclassifications for a given language and for each observation in the misclassification set, we iterated through each token to check if the token is in English. In this way, we counted the number of observations that contained at least one non-English word and divided that with the size of the misclassifications set. We used this ratio to investigate the effect of machine translation errors.", + "We found that 25.84% of Dutch, 21.76% of Turkish, 24.46% Spanish, and 10.71% of Russian reviews that were misclassified had non-English words in them. These non-English words might be causing the misclassifications. However, a large portion of the missclassifications is not caused due to not-translated words. At the end, the machine translation errors has some but not noticeable effects on our model. Therefore, we can claim that machine translation preserves most of the information necessary for sentiment analysis.", + "We also evaluated our model with an English corpus BIBREF27 to see its performance without any interference from machine translation errors. Using the English data for testing, the model achieved 87.06% accuracy where a majority baseline was 68.37% and the lexicon-based baseline was 60.10%.", + "Considering the improvements over the majority baseline achieved by the RNN model for both non-English (on the average 22.76% relative improvement; 15.82% relative improvement on Spanish, 72.71% vs. 84.21%, 30.53% relative improvement on Turkish, 56.97% vs. 74.36%, 37.13% relative improvement on Dutch, 59.63% vs. 81.77%, and 7.55% relative improvement on Russian, 79.60% vs. 85.62%) and English test sets (27.34% relative improvement), we can draw the conclusion that our model is robust to handle multiple languages. Building separate models for each language requires both labeled and unlabeled data. Even though having lots of labeled data in every language is the perfect case, it is unrealistic. Therefore, eliminating the resource requirement in this resource-constrained task is crucial. The fact that machine translation can be used in reusing models from different languages is promising for reducing the data requirements." + ] + }, + { + "section_name": "Conclusion", + "paragraphs": [ + "Building effective machine learning models for text requires data and different resources such as pre-trained word embeddings and reusable lexicons. Unfortunately, most of these resources are not entirely transferable to different domains, tasks or languages. Sentiment analysis is one such task that requires additional effort to transfer knowledge between languages.", + "In this paper, we studied the research question: Can we build reusable sentiment analysis models that can be utilized for making inferences in different languages without requiring separate models and resources for each language? To that end, we built a recurrent neural network model in the language that had largest data available. We took a general-to-specific model building strategy where the larger corpus that had reviews from different domains was first used to train the RNN model and a smaller single-domain corpus of sentiment reviews was used to specialize the model on the given domain. During scoring time, we used corpora for the given domain in different languages and translated them to English to be able to classify sentiments with the trained model. Experimental results showed that the proposed multilingual approach outperforms both the majority baseline and the lexicon-based baseline.", + "In this paper we made the sentiment analysis model specific to a single domain. For future work, we would like to investigate the effectiveness of our model on different review domains including hotel reviews and on different problems such as detecting stance." + ] + } + ], + "qas": [ + { + "question": "which non-english language had the best performance?", + "question_id": "e79a5b6b6680bd2f63e9f4adbaae1d7795d81e38", + "nlp_background": "", + "topic_background": "", + "paper_read": "", + "search_query": "", + "question_writer": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "Russian" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "Considering the improvements over the majority baseline achieved by the RNN model for both non-English (on the average 22.76% relative improvement; 15.82% relative improvement on Spanish, 72.71% vs. 84.21%, 30.53% relative improvement on Turkish, 56.97% vs. 74.36%, 37.13% relative improvement on Dutch, 59.63% vs. 81.77%, and 7.55% relative improvement on Russian, 79.60% vs. 85.62%) and English test sets (27.34% relative improvement), we can draw the conclusion that our model is robust to handle multiple languages. Building separate models for each language requires both labeled and unlabeled data. Even though having lots of labeled data in every language is the perfect case, it is unrealistic. Therefore, eliminating the resource requirement in this resource-constrained task is crucial. The fact that machine translation can be used in reusing models from different languages is promising for reducing the data requirements." + ], + "highlighted_evidence": [ + "Considering the improvements over the majority baseline achieved by the RNN model for both non-English (on the average 22.76% relative improvement; 15.82% relative improvement on Spanish, 72.71% vs. 84.21%, 30.53% relative improvement on Turkish, 56.97% vs. 74.36%, 37.13% relative improvement on Dutch, 59.63% vs. 81.77%, and 7.55% relative improvement on Russian, 79.60% vs. 85.62%) and English test sets (27.34% relative improvement), we can draw the conclusion that our model is robust to handle multiple languages." + ] + }, + "annotation_id": "2b2fe9e33cc5cb59f0f9504872a58e5e3828086e", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "Russsian", + "evidence": [ + "FLOAT SELECTED: Table 3: Accuracy results (%) for RNN-based approach compared with majority baseline and lexicon-based baseline." + ], + "highlighted_evidence": [ + "FLOAT SELECTED: Table 3: Accuracy results (%) for RNN-based approach compared with majority baseline and lexicon-based baseline." + ] + }, + "annotation_id": "d22c8180650a20e95c8b19eb364ff3224bb4d567", + "worker_id": "34c35a1877e453ecaebcf625df3ef788e1953cc4" + } + ] + }, + { + "question": "which non-english language was the had the worst results?", + "question_id": "c7486d039304ca9d50d0571236429f4f6fbcfcf7", + "nlp_background": "", + "topic_background": "", + "paper_read": "", + "search_query": "", + "question_writer": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "Turkish" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "Considering the improvements over the majority baseline achieved by the RNN model for both non-English (on the average 22.76% relative improvement; 15.82% relative improvement on Spanish, 72.71% vs. 84.21%, 30.53% relative improvement on Turkish, 56.97% vs. 74.36%, 37.13% relative improvement on Dutch, 59.63% vs. 81.77%, and 7.55% relative improvement on Russian, 79.60% vs. 85.62%) and English test sets (27.34% relative improvement), we can draw the conclusion that our model is robust to handle multiple languages. Building separate models for each language requires both labeled and unlabeled data. Even though having lots of labeled data in every language is the perfect case, it is unrealistic. Therefore, eliminating the resource requirement in this resource-constrained task is crucial. The fact that machine translation can be used in reusing models from different languages is promising for reducing the data requirements." + ], + "highlighted_evidence": [ + "Considering the improvements over the majority baseline achieved by the RNN model for both non-English (on the average 22.76% relative improvement; 15.82% relative improvement on Spanish, 72.71% vs. 84.21%, 30.53% relative improvement on Turkish, 56.97% vs. 74.36%, 37.13% relative improvement on Dutch, 59.63% vs. 81.77%, and 7.55% relative improvement on Russian, 79.60% vs. 85.62%) and English test sets (27.34% relative improvement), we can draw the conclusion that our model is robust to handle multiple languages." + ] + }, + "annotation_id": "0e6e41b3faecd132893b3ec7a3e3972b613a2c90", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + } + ] + }, + { + "question": "what datasets were used in evaluation?", + "question_id": "f1f1dcc67b3e4d554bfeb508226cdadb3c32d2e9", + "nlp_background": "", + "topic_background": "", + "paper_read": "", + "search_query": "", + "question_writer": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "SemEval-2016 Challenge Task 5 BIBREF27 , BIBREF28" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "For evaluation of the multilingual approach, we use four languages. These datasets are part of SemEval-2016 Challenge Task 5 BIBREF27 , BIBREF28 . Table TABREF7 shows the number of observations in each test corpus." + ], + "highlighted_evidence": [ + "These datasets are part of SemEval-2016 Challenge Task 5 BIBREF27 , BIBREF28 . Table TABREF7 shows the number of observations in each test corpus." + ] + }, + "annotation_id": "2bb929accb1fccc2be5c8903d9d02a7160bdf61a", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + " English reviews ", + " restaurant reviews from four different languages (Spanish, Turkish, Dutch, Russian)" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "Two sets of corpora are used in this study, both are publicly available. The first set consists of English reviews and the second set contains restaurant reviews from four different languages (Spanish, Turkish, Dutch, Russian). We focus on polarity detection in reviews, therefore all datasets in this study have two class values (positive, negative)." + ], + "highlighted_evidence": [ + "Two sets of corpora are used in this study, both are publicly available. The first set consists of English reviews and the second set contains restaurant reviews from four different languages (Spanish, Turkish, Dutch, Russian)." + ] + }, + "annotation_id": "34575f1d175be1c68aa5312e497e8792cb3a75e1", + "worker_id": "34c35a1877e453ecaebcf625df3ef788e1953cc4" + } + ] + }, + { + "question": "what are the baselines?", + "question_id": "a103636c8d1dbfa53341133aeb751ffec269415c", + "nlp_background": "", + "topic_background": "", + "paper_read": "", + "search_query": "", + "question_writer": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "majority baseline", + "lexicon-based approach" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "In addition to the majority baseline, we also compare our results with a lexicon-based approach. We use SentiWordNet BIBREF29 to obtain a positive and a negative sentiment score for each token in a review. Then sum of positive sentiment scores and negative sentiment scores for each review is obtained by summing up the scores for each token. If the positive sum score for a given review is greater than the negative sum score, we accept that review as a positive review. If negative sum is larger than or equal to the positive sum, the review is labeled as a negative review.", + "For experimental results, we report majority baseline for each language where the majority baseline corresponds to a model's accuracy if it always predicts the majority class in the dataset. For example, if the dataset has 60% of all reviews positive and 40% negative, majority baseline would be 60% because a model that always predicts “positive” will be 60% accurate and will make mistakes 40% of the time." + ], + "highlighted_evidence": [ + "In addition to the majority baseline, we also compare our results with a lexicon-based approach.", + "For experimental results, we report majority baseline for each language where the majority baseline corresponds to a model's accuracy if it always predicts the majority class in the dataset." + ] + }, + "annotation_id": "11cfaadad843b87f7fb0c872103c52ddb35d143f", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "majority baseline corresponds to a model's accuracy if it always predicts the majority class in the dataset", + "lexicon-based approach" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "For experimental results, we report majority baseline for each language where the majority baseline corresponds to a model's accuracy if it always predicts the majority class in the dataset. For example, if the dataset has 60% of all reviews positive and 40% negative, majority baseline would be 60% because a model that always predicts “positive” will be 60% accurate and will make mistakes 40% of the time.", + "In addition to the majority baseline, we also compare our results with a lexicon-based approach. We use SentiWordNet BIBREF29 to obtain a positive and a negative sentiment score for each token in a review. Then sum of positive sentiment scores and negative sentiment scores for each review is obtained by summing up the scores for each token. If the positive sum score for a given review is greater than the negative sum score, we accept that review as a positive review. If negative sum is larger than or equal to the positive sum, the review is labeled as a negative review." + ], + "highlighted_evidence": [ + "For experimental results, we report majority baseline for each language where the majority baseline corresponds to a model's accuracy if it always predicts the majority class in the dataset. For example, if the dataset has 60% of all reviews positive and 40% negative, majority baseline would be 60% because a model that always predicts “positive” will be 60% accurate and will make mistakes 40% of the time.\n\nIn addition to the majority baseline, we also compare our results with a lexicon-based approach. " + ] + }, + "annotation_id": "f70ac58596c06b4d0ddfc57c1170b03902897ee3", + "worker_id": "34c35a1877e453ecaebcf625df3ef788e1953cc4" + } + ] + }, + { + "question": "how did the authors translate the reviews to other languages?", + "question_id": "55139fcfe04ce90aad407e2e5a0067a45f31e07e", + "nlp_background": "", + "topic_background": "", + "paper_read": "", + "search_query": "", + "question_writer": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "Using Google translation API.", + "evidence": [ + "In order to eliminate the need to find data and build separate models for each language, we propose a multilingual approach where a single model is built in the language where the largest resources are available. In this paper we focus on English as there are several sentiment analysis datasets in English. To make the English sentiment analysis model as generalizable as possible, we first start by training with a large dataset that has product reviews for different categories. Then, using the trained weights from the larger generic dataset, we make the model more specialized for a specific domain. We further train the model with domain-specific English reviews and use this trained model to score reviews that share the same domain from different languages. To be able to employ the trained model, test sets are first translated to English via machine translation and then inference takes place. Figure FIGREF1 shows our multilingual sentiment analysis approach. It is important to note that this approach does not utilize any resource in any of the languages of the test sets (e.g., word embeddings, lexicons, training set).", + "Throughout our experiments, we use SAS Deep Learning Toolkit. For machine translation, Google translation API is used." + ], + "highlighted_evidence": [ + " To be able to employ the trained model, test sets are first translated to English via machine translation and then inference takes place. ", + " For machine translation, Google translation API is used." + ] + }, + "annotation_id": "b39ff9b1051500dac6197ca9535ec3182267c88b", + "worker_id": "34c35a1877e453ecaebcf625df3ef788e1953cc4" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "Google translation API" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "Throughout our experiments, we use SAS Deep Learning Toolkit. For machine translation, Google translation API is used." + ], + "highlighted_evidence": [ + "For machine translation, Google translation API is used." + ] + }, + "annotation_id": "dccccc28dadf1497243498e6b66dedc901795d03", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + } + ] + }, + { + "question": "what dataset was used for training?", + "question_id": "fbaf060004f196a286fef67593d2d76826f0304e", + "nlp_background": "", + "topic_background": "", + "paper_read": "", + "search_query": "", + "question_writer": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "Amazon reviews", + "Yelp restaurant reviews", + "restaurant reviews" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "With the goal of building a generalizable sentiment analysis model, we used three different training sets as provided in Table TABREF5 . One of these three datasets (Amazon reviews BIBREF23 , BIBREF24 ) is larger and has product reviews from several different categories including book reviews, electronics products reviews, and application reviews. The other two datasets are to make the model more specialized in the domain. In this paper we focus on restaurant reviews as our domain and use Yelp restaurant reviews dataset extracted from Yelp Dataset Challenge BIBREF25 and restaurant reviews dataset as part of a Kaggle competition BIBREF26 ." + ], + "highlighted_evidence": [ + "With the goal of building a generalizable sentiment analysis model, we used three different training sets as provided in Table TABREF5 . One of these three datasets (Amazon reviews BIBREF23 , BIBREF24 ) is larger and has product reviews from several different categories including book reviews, electronics products reviews, and application reviews. The other two datasets are to make the model more specialized in the domain. In this paper we focus on restaurant reviews as our domain and use Yelp restaurant reviews dataset extracted from Yelp Dataset Challenge BIBREF25 and restaurant reviews dataset as part of a Kaggle competition BIBREF26 ." + ] + }, + "annotation_id": "86952000b285e7f0a6a4b8bd22314fdcef431a52", + "worker_id": "34c35a1877e453ecaebcf625df3ef788e1953cc4" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "Amazon reviews BIBREF23 , BIBREF24", + "Yelp restaurant reviews dataset", + " restaurant reviews dataset as part of a Kaggle competition BIBREF26" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "With the goal of building a generalizable sentiment analysis model, we used three different training sets as provided in Table TABREF5 . One of these three datasets (Amazon reviews BIBREF23 , BIBREF24 ) is larger and has product reviews from several different categories including book reviews, electronics products reviews, and application reviews. The other two datasets are to make the model more specialized in the domain. In this paper we focus on restaurant reviews as our domain and use Yelp restaurant reviews dataset extracted from Yelp Dataset Challenge BIBREF25 and restaurant reviews dataset as part of a Kaggle competition BIBREF26 ." + ], + "highlighted_evidence": [ + "With the goal of building a generalizable sentiment analysis model, we used three different training sets as provided in Table TABREF5 . One of these three datasets (Amazon reviews BIBREF23 , BIBREF24 ) is larger and has product reviews from several different categories including book reviews, electronics products reviews, and application reviews. The other two datasets are to make the model more specialized in the domain. In this paper we focus on restaurant reviews as our domain and use Yelp restaurant reviews dataset extracted from Yelp Dataset Challenge BIBREF25 and restaurant reviews dataset as part of a Kaggle competition BIBREF26 ." + ] + }, + "annotation_id": "bdd840844b92e11905eeef5524fc529e2bf7fb47", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + } + ] + } + ], + "figures_and_tables": [ + { + "file": "2-Figure1-1.png", + "caption": "Figure 1: Multilingual sentiment analysis approach." + }, + { + "file": "3-Table2-1.png", + "caption": "Table 2: Datasets used for testing." + }, + { + "file": "3-Figure2-1.png", + "caption": "Figure 2: Training sentiment analysis model with RNN." + }, + { + "file": "3-Table3-1.png", + "caption": "Table 3: Accuracy results (%) for RNN-based approach compared with majority baseline and lexicon-based baseline." + }, + { + "file": "3-Table1-1.png", + "caption": "Table 1: Datasets used for training." + }, + { + "file": "3-Table4-1.png", + "caption": "Table 4: Percentage of false positives and false negatives of wrong classi cations." + }, + { + "file": "4-Figure3-1.png", + "caption": "Figure 3: Multiple comparisons between majority baseline, lexicon-based baseline and RNN." + }, + { + "file": "4-Table5-1.png", + "caption": "Table 5: Multiple comparison of means." + } + ] + }, + "2002.02070": { + "title": "Understanding Car-Speak: Replacing Humans in Dealerships", + "abstract": "A large portion of the car-buying experience in the United States involves interactions at a car dealership. At the dealership, the car-buyer relays their needs to a sales representative. However, most car-buyers are only have an abstract description of the vehicle they need. Therefore, they are only able to describe their ideal car in \"car-speak\". Car-speak is abstract language that pertains to a car's physical attributes. In this paper, we define car-speak. We also aim to curate a reasonable data set of car-speak language. Finally, we train several classifiers in order to classify car-speak.", + "full_text": [ + { + "section_name": "Introduction", + "paragraphs": [ + "A large portion of the car-buying experience in the United States involves interactions at a car dealership BIBREF0, BIBREF1, BIBREF2. Traditionally, a car dealer listens and understands the needs of the client and helps them find what car is right based on their needs.", + "With the advent of the internet, many potential car buyers take to the web to research cars before going to a dealership in person BIBREF0, BIBREF2. However, nearly 50% of customers bought a car at the dealership based on the sales representative's advice, not their own research BIBREF1, BIBREF2.", + "Throughout this interaction the dealer is acting as a type of translator or classifier. The dealer takes a natural language input (e.g. “I need a fast, family friendly, reliable car under $20k”) and returns a list of suggestions. The dealer understands the ideas of “fast”, “family friendly”, and “reliable” and is able to come up with a reasonable recommendation based on this knowledge.", + "In this paper we aim to create a system that can understand car-speak based on some natural language input (we want to recreate the dealer from above). But how do we prepare a proper training set for a Natural Language model? What model is best suited to this problem? Can this model take a human out of the car-buying process? To answer these questions, the remainder of this paper makes the following contributions:", + "Defining “car-speak” and its role in the car-buying process.", + "Appropriate training data for a Natural Language model.", + "A model that is able to properly classify car-speak and return a car.", + "We aim to accomplish these goals in a scientific manner, using real data and modern methods." + ] + }, + { + "section_name": "Related Work", + "paragraphs": [ + "There has been some work done in the field of car-sales and dealer interactions. However, this is the first work that specifically focuses on the", + "Deloitte has published a report on the entire car-buying process BIBREF0. The report goes into great depth about the methods potential buyers use to find new cars to buy, and how they go about buying them. The report tells us that there are several unique phases that a potential buyer goes through before buying a car.", + "Verhoef et al. looked at the specifics of dealer interaction and how dealers retain customers BIBREF3. Verhoef tells us how important dealers are to the car-buying process. He also explains how influential a dealer can be on what car the buyer purchases.", + "Jeff Kershner compiled a series of statistics about Dealership Sales BIBREF1. These statistics focus on small social interactions BIBREF4 between the dealer and the buyer.", + "Barley explains the increasing role of technology in the car-buying process BIBREF2. Barley tells us that users prefer to use technology/robots to find the cars they want to buy instead of going to a dealer, due the distrust towards sales representatives." + ] + }, + { + "section_name": "What is Car-speak?", + "paragraphs": [ + "When a potential buyer begins to identify their next car-purchase they begin with identifying their needs. These needs often come in the form of an abstract situation, for instance, “I need a car that goes really fast”. This could mean that they need a car with a V8 engine type or a car that has 500 horsepower, but the buyer does not know that, all they know is that they need a “fast” car.", + "The term “fast” is car-speak. Car-speak is abstract language that pertains to a car's physical attribute(s). In this instance the physical attributes that the term “fast” pertains to could be the horsepower, or it could be the car's form factor (how the car looks). However, we do not know exactly which attributes the term “fast” refers to.", + "The use of car-speak is present throughout the car-buying process. It begins in the Research Phase where buyers identify their needs BIBREF0. When the buyer goes to a dealership to buy a car, they communicate with the dealer in similar car-speak BIBREF2 and convey their needs to the sales representative. Finally, the sales representative uses their internal classifier to translate this car-speak into actual physical attributes (e.g. `fast' $ \\longrightarrow $ `700 horsepower & a sleek form factor') and offers a car to the buyer.", + "Understanding car-speak is not a trivial task. Figure FIGREF4 shows two cars that have high top speeds, however both cars may not be considered “fast”. We need to mine the ideas that people have about cars in order to determine which cars are “fast” and which cars are not." + ] + }, + { + "section_name": "Gathering Car-speak Data", + "paragraphs": [ + "We aim to curate a data set of car-speak in order to train a model properly. However, there are a few challenges that present themselves: What is a good source of car-speak? How can we acquire the data? How can we be sure the data set is relevant?", + "What is a good source of car-speak? We find plenty of car-speak in car reviews. Table TABREF5 provides excerpts from reviews with the car-speak terms bolded. Car reviews often describe cars in an abstract manner, which makes the review more useful for car-buyers. The reviews are often also about specific use-cases for each car (e.g. using the car to tow a trailer), so they capture all possible aspects of a car. The reviews are each written about a specific car, so we are able to map car-speak to a specific car model.", + "We choose the reviews from the U.S. News & World Report because they have easily accessible full-length reviews about every car that has been sold in the United States since 2006 BIBREF5.", + "How can we acquire the data? We can acquire this data using modern web-scraping tools such as beautiful-soup. The data is publicly available on https://cars.usnews.com/cars-trucks BIBREF5. These reviews also include a scorecard and justification of their reviews.", + "How can we be sure the data set is relevant? On average vehicles on United States roads are 11.6 years old, making the average manufacturing year 2006-2007 BIBREF6, BIBREF7. In order to have a relevant data set we gather all of the available reviews for car models made between the years 2000 and 2018." + ] + }, + { + "section_name": "Translating Car-Speak", + "paragraphs": [ + "Our data set contains $3,209$ reviews about 553 different cars from 49 different car manufacturers. In order to accomplish our goal of translating and classifying car-speak we need to filter our data set so that we only have the most relevant terms. We then need to be able to weight each word in each review, so that we can determine the most relevant ideas in each document for the purpose of classification. Finally, we need to train various classification models and evaluate them." + ] + }, + { + "section_name": "Translating Car-Speak ::: Filtering the Data", + "paragraphs": [ + "We would like to be able to represent each car with the most relevant car-speak terms. We can do this by filtering each review using the NLTK library BIBREF8, only retaining the most relevant words. First we token-ize each review and then keep only the nouns and adjectives from each review since they are the most salient parts of speech BIBREF9. This leaves us with $10,867$ words across all reviews. Figure FIGREF6 shows the frequency of the top 20 words that remain.", + "Words such as “saftey” and “luxury” are among the top words used in reviews. These words are very good examples of car-speak. Both words are abstract descriptions of cars, but both have physical characteristics that are associated with them as we discussed in Section SECREF3." + ] + }, + { + "section_name": "Translating Car-Speak ::: TF-IDF", + "paragraphs": [ + "So far we have compiled the most relevant terms in from the reviews. We now need to weight these terms for each review, so that we know the car-speak terms are most associated with a car. Using TF-IDF (Term Frequency-Inverse Document Frequency) has been used as a reliable metric for finding the relevant terms in a document BIBREF10.", + "We represent each review as a vector of TF-IDF scores for each word in the review. The length of this vector is $10,867$. We label each review vector with the car it reviews. We ignore the year of the car being reviewed and focus specifically on the model (i.e Acura ILX, not 2013 Acura ILX). This is because there a single model of car generally retains the same characteristics over time BIBREF11, BIBREF12." + ] + }, + { + "section_name": "Translating Car-Speak ::: Classification Experiments", + "paragraphs": [ + "We train a series of classifiers in order to classify car-speak. We train three classifiers on the review vectors that we prepared in Section SECREF8. The classifiers we use are K Nearest Neighbors (KNN), Random Forest (RF), Support Vector Machine (SVM), and Multi-layer Perceptron (MLP) BIBREF13.", + "In order to evaluate our classifiers, we perform 4-fold cross validation on a shuffled data set. Table TABREF10 shows the F1 micro and F1 macro scores for all the classifiers. The KNN classifier seem to perform the best across all four metrics. This is probably due to the multi-class nature of the data set." + ] + }, + { + "section_name": "Conclusion & Future Work", + "paragraphs": [ + "In this paper we aim to provide an introductory understanding of car-speak and a way to automate car dealers at dealerships. We first provide a definition of “car-speak” in Section SECREF3. We explore what constitutes car-speak and how to identify car-speak.", + "We also gather a data set of car-speak to use for exploration and training purposes. This data set id full of vehicle reviews from U.S. News BIBREF5. These reviews provide a reasonable set of car-speak data that we can study.", + "Finally, we create and test several classifiers that are trained on the data we gathered. While these classifiers did not perform particularly well, they provide a good starting point for future work on this subject.", + "In the future we plan to use more complex models to attempt to understand car-speak. We also would like to test our classifiers on user-provided natural language queries. This would be a more practical evaluation of our classification. It would also satisfy the need for a computer system that understands car-speak." + ] + } + ], + "qas": [ + { + "question": "Is car-speak language collection of abstract features that classifier is later trained on?", + "question_id": "25c1c4a91f5dedd4e06d14121af3b5921db125e9", + "nlp_background": "zero", + "topic_background": "unfamiliar", + "paper_read": "no", + "search_query": "", + "question_writer": "258ee4069f740c400c0049a2580945a1cc7f044c", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": false, + "free_form_answer": "", + "evidence": [ + "The term “fast” is car-speak. Car-speak is abstract language that pertains to a car's physical attribute(s). In this instance the physical attributes that the term “fast” pertains to could be the horsepower, or it could be the car's form factor (how the car looks). However, we do not know exactly which attributes the term “fast” refers to.", + "We train a series of classifiers in order to classify car-speak. We train three classifiers on the review vectors that we prepared in Section SECREF8. The classifiers we use are K Nearest Neighbors (KNN), Random Forest (RF), Support Vector Machine (SVM), and Multi-layer Perceptron (MLP) BIBREF13." + ], + "highlighted_evidence": [ + "Car-speak is abstract language that pertains to a car's physical attribute(s). In this instance the physical attributes that the term “fast” pertains to could be the horsepower, or it could be the car's form factor (how the car looks). However, we do not know exactly which attributes the term “fast” refers to.", + "We train a series of classifiers in order to classify car-speak. We train three classifiers on the review vectors that we prepared in Section SECREF8. The classifiers we use are K Nearest Neighbors (KNN), Random Forest (RF), Support Vector Machine (SVM), and Multi-layer Perceptron (MLP) BIBREF13." + ] + }, + "annotation_id": "57c005faa3e141d0520a5c5e39a7ae074005b87d", + "worker_id": "a0b403873302db7cada39008f04d01155ef68f4f" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": false, + "free_form_answer": "", + "evidence": [ + "The term “fast” is car-speak. Car-speak is abstract language that pertains to a car's physical attribute(s). In this instance the physical attributes that the term “fast” pertains to could be the horsepower, or it could be the car's form factor (how the car looks). However, we do not know exactly which attributes the term “fast” refers to.", + "Our data set contains $3,209$ reviews about 553 different cars from 49 different car manufacturers. In order to accomplish our goal of translating and classifying car-speak we need to filter our data set so that we only have the most relevant terms. We then need to be able to weight each word in each review, so that we can determine the most relevant ideas in each document for the purpose of classification. Finally, we need to train various classification models and evaluate them." + ], + "highlighted_evidence": [ + "The term “fast” is car-speak. Car-speak is abstract language that pertains to a car's physical attribute(s). In this instance the physical attributes that the term “fast” pertains to could be the horsepower, or it could be the car's form factor (how the car looks). However, we do not know exactly which attributes the term “fast” refers to.", + "In order to accomplish our goal of translating and classifying car-speak we need to filter our data set so that we only have the most relevant terms. We then need to be able to weight each word in each review, so that we can determine the most relevant ideas in each document for the purpose of classification. Finally, we need to train various classification models and evaluate them." + ] + }, + "annotation_id": "e501cac4a83ea9656f60fe19999bad570c02506d", + "worker_id": "34c35a1877e453ecaebcf625df3ef788e1953cc4" + } + ] + }, + { + "question": "Is order of \"words\" important in car speak language?", + "question_id": "f88036174b4a0dbf4fe70ddad884d16082c5748d", + "nlp_background": "zero", + "topic_background": "unfamiliar", + "paper_read": "no", + "search_query": "", + "question_writer": "258ee4069f740c400c0049a2580945a1cc7f044c", + "answers": [ + { + "answer": { + "unanswerable": true, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "", + "evidence": [], + "highlighted_evidence": [] + }, + "annotation_id": "531f13e3a65795200fc5ab0bb8645dbf0df280ee", + "worker_id": "34c35a1877e453ecaebcf625df3ef788e1953cc4" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": false, + "free_form_answer": "", + "evidence": [ + "We would like to be able to represent each car with the most relevant car-speak terms. We can do this by filtering each review using the NLTK library BIBREF8, only retaining the most relevant words. First we token-ize each review and then keep only the nouns and adjectives from each review since they are the most salient parts of speech BIBREF9. This leaves us with $10,867$ words across all reviews. Figure FIGREF6 shows the frequency of the top 20 words that remain.", + "So far we have compiled the most relevant terms in from the reviews. We now need to weight these terms for each review, so that we know the car-speak terms are most associated with a car. Using TF-IDF (Term Frequency-Inverse Document Frequency) has been used as a reliable metric for finding the relevant terms in a document BIBREF10." + ], + "highlighted_evidence": [ + " We can do this by filtering each review using the NLTK library BIBREF8, only retaining the most relevant words. First we token-ize each review and then keep only the nouns and adjectives from each review since they are the most salient parts of speech BIBREF9. This leaves us with $10,867$ words across all reviews.", + "Using TF-IDF (Term Frequency-Inverse Document Frequency) has been used as a reliable metric for finding the relevant terms in a document BIBREF10." + ] + }, + "annotation_id": "ea4e243641c657322b71a2a4fb70a527fa7f87e7", + "worker_id": "a0b403873302db7cada39008f04d01155ef68f4f" + } + ] + }, + { + "question": "What are labels in car speak language dataset?", + "question_id": "a267d620af319b48e56c191aa4c433ea3870f6fb", + "nlp_background": "zero", + "topic_background": "unfamiliar", + "paper_read": "no", + "search_query": "", + "question_writer": "258ee4069f740c400c0049a2580945a1cc7f044c", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "car " + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "We represent each review as a vector of TF-IDF scores for each word in the review. The length of this vector is $10,867$. We label each review vector with the car it reviews. We ignore the year of the car being reviewed and focus specifically on the model (i.e Acura ILX, not 2013 Acura ILX). This is because there a single model of car generally retains the same characteristics over time BIBREF11, BIBREF12." + ], + "highlighted_evidence": [ + "We label each review vector with the car it reviews. " + ] + }, + "annotation_id": "820cf6fc2076beb25c8bb7c6838415265dc8f40b", + "worker_id": "34c35a1877e453ecaebcf625df3ef788e1953cc4" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "the car" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "Our data set contains $3,209$ reviews about 553 different cars from 49 different car manufacturers. In order to accomplish our goal of translating and classifying car-speak we need to filter our data set so that we only have the most relevant terms. We then need to be able to weight each word in each review, so that we can determine the most relevant ideas in each document for the purpose of classification. Finally, we need to train various classification models and evaluate them.", + "We represent each review as a vector of TF-IDF scores for each word in the review. The length of this vector is $10,867$. We label each review vector with the car it reviews. We ignore the year of the car being reviewed and focus specifically on the model (i.e Acura ILX, not 2013 Acura ILX). This is because there a single model of car generally retains the same characteristics over time BIBREF11, BIBREF12." + ], + "highlighted_evidence": [ + "Our data set contains $3,209$ reviews about 553 different cars from 49 different car manufacturers. In order to accomplish our goal of translating and classifying car-speak we need to filter our data set so that we only have the most relevant terms. We then need to be able to weight each word in each review, so that we can determine the most relevant ideas in each document for the purpose of classification.", + "We represent each review as a vector of TF-IDF scores for each word in the review. The length of this vector is $10,867$. We label each review vector with the car it reviews. We ignore the year of the car being reviewed and focus specifically on the model (i.e Acura ILX, not 2013 Acura ILX). " + ] + }, + "annotation_id": "dc1a8b8cb6334eef875752b078f005fa3826a25b", + "worker_id": "a0b403873302db7cada39008f04d01155ef68f4f" + } + ] + }, + { + "question": "How big is dataset of car-speak language?", + "question_id": "899ed05c460bf2aa0aa65101cad1986d4f622652", + "nlp_background": "zero", + "topic_background": "unfamiliar", + "paper_read": "no", + "search_query": "", + "question_writer": "258ee4069f740c400c0049a2580945a1cc7f044c", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "$3,209$ reviews " + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "Our data set contains $3,209$ reviews about 553 different cars from 49 different car manufacturers. In order to accomplish our goal of translating and classifying car-speak we need to filter our data set so that we only have the most relevant terms. We then need to be able to weight each word in each review, so that we can determine the most relevant ideas in each document for the purpose of classification. Finally, we need to train various classification models and evaluate them." + ], + "highlighted_evidence": [ + "Our data set contains $3,209$ reviews about 553 different cars from 49 different car manufacturers." + ] + }, + "annotation_id": "6b9b900a596cfe81f85eb4b016a78dcdcb661990", + "worker_id": "34c35a1877e453ecaebcf625df3ef788e1953cc4" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "$3,209$ reviews about 553 different cars from 49 different car manufacturers" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "Our data set contains $3,209$ reviews about 553 different cars from 49 different car manufacturers. In order to accomplish our goal of translating and classifying car-speak we need to filter our data set so that we only have the most relevant terms. We then need to be able to weight each word in each review, so that we can determine the most relevant ideas in each document for the purpose of classification. Finally, we need to train various classification models and evaluate them." + ], + "highlighted_evidence": [ + "Our data set contains $3,209$ reviews about 553 different cars from 49 different car manufacturers. In order to accomplish our goal of translating and classifying car-speak we need to filter our data set so that we only have the most relevant terms." + ] + }, + "annotation_id": "a7e44116c3d6871a537078734084e5b00bbe6ea8", + "worker_id": "a0b403873302db7cada39008f04d01155ef68f4f" + } + ] + }, + { + "question": "What is the performance of classifiers?", + "question_id": "d53299fac8c94bd0179968eb868506124af407d1", + "nlp_background": "zero", + "topic_background": "unfamiliar", + "paper_read": "no", + "search_query": "", + "question_writer": "258ee4069f740c400c0049a2580945a1cc7f044c", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "Table TABREF10", + " The KNN classifier seem to perform the best across all four metrics. This is probably due to the multi-class nature of the data set", + " While these classifiers did not perform particularly well, they provide a good starting point for future work on this subject" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "In order to evaluate our classifiers, we perform 4-fold cross validation on a shuffled data set. Table TABREF10 shows the F1 micro and F1 macro scores for all the classifiers. The KNN classifier seem to perform the best across all four metrics. This is probably due to the multi-class nature of the data set.", + "FLOAT SELECTED: Table 2: Evaluation metrics for all classifiers." + ], + "highlighted_evidence": [ + "In order to evaluate our classifiers, we perform 4-fold cross validation on a shuffled data set. Table TABREF10 shows the F1 micro and F1 macro scores for all the classifiers. The KNN classifier seem to perform the best across all four metrics. This is probably due to the multi-class nature of the data set.", + "FLOAT SELECTED: Table 2: Evaluation metrics for all classifiers." + ] + }, + "annotation_id": "e7afe3d129c67b64dfa3c7c742caab4b969b3ccb", + "worker_id": "a0b403873302db7cada39008f04d01155ef68f4f" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "Using F1 Micro measure, the KNN classifier perform 0.6762, the RF 0.6687, SVM 0.6712 and MLP 0.6778.", + "evidence": [ + "FLOAT SELECTED: Table 2: Evaluation metrics for all classifiers." + ], + "highlighted_evidence": [ + "FLOAT SELECTED: Table 2: Evaluation metrics for all classifiers." + ] + }, + "annotation_id": "f5607a22d620c9a0858a29d2444d1981950eb6fe", + "worker_id": "34c35a1877e453ecaebcf625df3ef788e1953cc4" + } + ] + }, + { + "question": "What classifiers have been trained?", + "question_id": "29f2954098f055fb19d9502572f085862d75bf61", + "nlp_background": "zero", + "topic_background": "unfamiliar", + "paper_read": "no", + "search_query": "", + "question_writer": "258ee4069f740c400c0049a2580945a1cc7f044c", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "KNN\nRF\nSVM\nMLP", + "evidence": [ + "FLOAT SELECTED: Table 2: Evaluation metrics for all classifiers.", + "In order to evaluate our classifiers, we perform 4-fold cross validation on a shuffled data set. Table TABREF10 shows the F1 micro and F1 macro scores for all the classifiers. The KNN classifier seem to perform the best across all four metrics. This is probably due to the multi-class nature of the data set." + ], + "highlighted_evidence": [ + "FLOAT SELECTED: Table 2: Evaluation metrics for all classifiers.", + "In order to evaluate our classifiers, we perform 4-fold cross validation on a shuffled data set. Table TABREF10 shows the F1 micro and F1 macro scores for all the classifiers." + ] + }, + "annotation_id": "23a17e180c21e2038c7158191379c6a4b6cb48b7", + "worker_id": "a0b403873302db7cada39008f04d01155ef68f4f" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + " K Nearest Neighbors (KNN)", + "Random Forest (RF)", + "Support Vector Machine (SVM)", + "Multi-layer Perceptron (MLP)" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "We train a series of classifiers in order to classify car-speak. We train three classifiers on the review vectors that we prepared in Section SECREF8. The classifiers we use are K Nearest Neighbors (KNN), Random Forest (RF), Support Vector Machine (SVM), and Multi-layer Perceptron (MLP) BIBREF13." + ], + "highlighted_evidence": [ + " The classifiers we use are K Nearest Neighbors (KNN), Random Forest (RF), Support Vector Machine (SVM), and Multi-layer Perceptron (MLP) BIBREF13." + ] + }, + "annotation_id": "94475d71ec9057a0abd22e684e725bf2e28e42c1", + "worker_id": "34c35a1877e453ecaebcf625df3ef788e1953cc4" + } + ] + }, + { + "question": "How does car speak pertains to a car's physical attributes?", + "question_id": "6bf93968110c6e3e3640360440607744007a5228", + "nlp_background": "zero", + "topic_background": "unfamiliar", + "paper_read": "no", + "search_query": "", + "question_writer": "258ee4069f740c400c0049a2580945a1cc7f044c", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "we do not know exactly" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "The term “fast” is car-speak. Car-speak is abstract language that pertains to a car's physical attribute(s). In this instance the physical attributes that the term “fast” pertains to could be the horsepower, or it could be the car's form factor (how the car looks). However, we do not know exactly which attributes the term “fast” refers to." + ], + "highlighted_evidence": [ + "Car-speak is abstract language that pertains to a car's physical attribute(s). In this instance the physical attributes that the term “fast” pertains to could be the horsepower, or it could be the car's form factor (how the car looks). However, we do not know exactly which attributes the term “fast” refers to." + ] + }, + "annotation_id": "b4858ea7b21f6705974b7e02189349ea82f9d719", + "worker_id": "34c35a1877e453ecaebcf625df3ef788e1953cc4" + } + ] + } + ], + "figures_and_tables": [ + { + "file": "1-Figure1-1.png", + "caption": "Figure 1: Both of these cars can achieve high speeds. Which is “fast”?" + }, + { + "file": "2-Table1-1.png", + "caption": "Table 1: Excerpts from car reviews." + }, + { + "file": "2-Figure2-1.png", + "caption": "Figure 2: The frequencies of the top 20 words in reviews." + }, + { + "file": "3-Table2-1.png", + "caption": "Table 2: Evaluation metrics for all classifiers." + } + ] + }, + "1912.08904": { + "title": "Macaw: An Extensible Conversational Information Seeking Platform", + "abstract": "Conversational information seeking (CIS) has been recognized as a major emerging research area in information retrieval. Such research will require data and tools, to allow the implementation and study of conversational systems. This paper introduces Macaw, an open-source framework with a modular architecture for CIS research. Macaw supports multi-turn, multi-modal, and mixed-initiative interactions, and enables research for tasks such as document retrieval, question answering, recommendation, and structured data exploration. It has a modular design to encourage the study of new CIS algorithms, which can be evaluated in batch mode. It can also integrate with a user interface, which allows user studies and data collection in an interactive mode, where the back end can be fully algorithmic or a wizard of oz setup. Macaw is distributed under the MIT License.", + "full_text": [ + { + "section_name": "Introduction", + "paragraphs": [ + "The rapid growth in speech and small screen interfaces, particularly on mobile devices, has significantly influenced the way users interact with intelligent systems to satisfy their information needs. The growing interest in personal digital assistants, such as Amazon Alexa, Apple Siri, Google Assistant, and Microsoft Cortana, demonstrates the willingness of users to employ conversational interactions BIBREF0. As a result, conversational information seeking (CIS) has been recognized as a major emerging research area in the Third Strategic Workshop on Information Retrieval (SWIRL 2018) BIBREF1.", + "Research progress in CIS relies on the availability of resources to the community. There have been recent efforts on providing data for various CIS tasks, such as the TREC 2019 Conversational Assistance Track (CAsT), MISC BIBREF2, Qulac BIBREF3, CoQA BIBREF4, QuAC BIBREF5, SCS BIBREF6, and CCPE-M BIBREF7. In addition, BIBREF8 have implemented a demonstration for conversational movie recommendation based on Google's DialogFlow. Despite all of these resources, the community still feels the lack of a suitable platform for developing CIS systems. We believe that providing such platform will speed up the progress in conversational information seeking research. Therefore, we developed a general framework for supporting CIS research. The framework is called Macaw. This paper describes the high-level architecture of Macaw, the supported functionality, and our future vision. Researchers working on various CIS tasks should be able to take advantage of Macaw in their projects.", + "Macaw is designed based on a modular architecture to support different information seeking tasks, including conversational search, conversational question answering, conversational recommendation, and conversational natural language interface to structured and semi-structured data. Each interaction in Macaw (from both user and system) is a Message object, thus a conversation is a list of Messages. Macaw consists of multiple actions, each action is a module that can satisfy the information needs of users for some requests. For example, search and question answering can be two actions in Macaw. Even multiple search algorithms can be also seen as multiple actions. Each action can produce multiple outputs (e.g., multiple retrieved documents). For every user interaction, Macaw runs all actions in parallel. The actions' outputs produced within a predefined time interval (i.e., an interaction timeout constant) are then post-processed. Macaw can choose one or combine multiple of these outputs and prepare an output Message object as the user's response.", + "The modular design of Macaw makes it relatively easy to configure a different user interface or add a new one. The current implementation of Macaw supports a command line interface as well as mobile, desktop, and web apps. In more detail, Macaw's interface can be a Telegram bot, which supports a wide range of devices and operating systems (see FIGREF4). This allows Macaw to support multi-modal interactions, such as text, speech, image, click, etc. A number of APIs for automatic speech recognition and generation have been employed to support speech interactions. Note that the Macaw's architecture and implementation allows mixed-initiative interactions.", + "The research community can benefit from Macaw for the following purposes:", + "[leftmargin=*]", + "Developing algorithms, tools, and techniques for CIS.", + "Studying user interactions with CIS systems.", + "Performing CIS studies based on an intermediary person and wizard of oz.", + "Preparing quick demonstration for a developed CIS model." + ] + }, + { + "section_name": "Macaw Architecture", + "paragraphs": [ + "Macaw has a modular design, with the goal of making it easy to configure and add new modules such as a different user interface or different retrieval module. The overall setup also follows a Model-View-Controller (MVC) like architecture. The design decisions have been made to smooth the Macaw's adoptions and extensions. Macaw is implemented in Python, thus machine learning models implemented using PyTorch, Scikit-learn, or TensorFlow can be easily integrated into Macaw. The high-level overview of Macaw is depicted in FIGREF8. The user interacts with the interface and the interface produces a Message object from the current interaction of user. The interaction can be in multi-modal form, such as text, speech, image, and click. Macaw stores all interactions in an “Interaction Database”. For every interaction, Macaw looks for most recent user-system interactions (including the system's responses) to create a list of Messages, called the conversation list. It is then dispatched to multiple information seeking (and related) actions. The actions run in parallel, and each should respond within a pre-defined time interval. The output selection component selects from (or potentially combines) the outputs generated by different actions and creates a Message object as the system's response. This message is logged into the interaction database and is sent to the interface to be presented to the user. Again, the response message can be multi-modal and include text, speech, link, list of options, etc.", + "Macaw also supports Wizard of Oz studies or intermediary-based information seeking studies. The architecture of Macaw for such setup is presented in FIGREF16. As shown in the figure, the seeker interacts with a real conversational interface that supports multi-modal and mixed-initiative interactions in multiple devices. The intermediary (or the wizard) receives the seeker's message and performs different information seeking actions with Macaw. All seeker-intermediary and intermediary-system interactions will be logged for further analysis. This setup can simulate an ideal CIS system and thus is useful for collecting high-quality data from real users for CIS research." + ] + }, + { + "section_name": "Retrieval and Question Answering in Macaw", + "paragraphs": [ + "The overview of retrieval and question answering actions in Macaw is shown in FIGREF17. These actions consist of the following components:", + "[leftmargin=*]", + "Co-Reference Resolution: To support multi-turn interactions, it is sometimes necessary to use co-reference resolution techniques for effective retrieval. In Macaw, we identify all the co-references from the last request of user to the conversation history. The same co-reference resolution outputs can be used for different query generation components. This can be a generic or action-specific component.", + "Query Generation: This component generates a query based on the past user-system interactions. The query generation component may take advantage of co-reference resolution for query expansion or re-writing.", + "Retrieval Model: This is the core ranking component that retrieves documents or passages from a large collection. Macaw can retrieve documents from an arbitrary document collection using the Indri python interface BIBREF9, BIBREF10. We also provide the support for web search using the Bing Web Search API. Macaw also allows multi-stage document re-ranking.", + "Result Generation: The retrieved documents can be too long to be presented using some interfaces. Result generation is basically a post-processing step ran on the retrieved result list. In case of question answering, it can employ answer selection or generation techniques, such as machine reading comprehension models. For example, Macaw features the DrQA model BIBREF11 for question answering.", + "These components are implemented in a generic form, so researchers can easily replace them with their own favorite algorithms." + ] + }, + { + "section_name": "User Interfaces", + "paragraphs": [ + "We have implemented the following interfaces for Macaw:", + "[leftmargin=*]", + "File IO: This interface is designed for experimental purposes, such as evaluating the performance of a conversational search technique on a dataset with multiple queries. This is not an interactive interface.", + "Standard IO: This interactive command line interface is designed for development purposes to interact with the system, see the logs, and debug or improve the system.", + "Telegram: This interactive interface is designed for interaction with real users (see FIGREF4). Telegram is a popular instant messaging service whose client-side code is open-source. We have implemented a Telegram bot that can be used with different devices (personal computers, tablets, and mobile phones) and different operating systems (Android, iOS, Linux, Mac OS, and Windows). This interface allows multi-modal interactions (text, speech, click, image). It can be also used for speech-only interactions. For speech recognition and generation, Macaw relies on online APIs, e.g., the services provided by Google Cloud and Microsoft Azure. In addition, there exist multiple popular groups and channels in Telegram, which allows further integration of social networks with conversational systems. For example, see the Naseri and Zamani's study on news popularity in Telegram BIBREF12.", + "Similar to the other modules, one can easily extend Macaw using other appropriate user interfaces." + ] + }, + { + "section_name": "Limitations and Future Work", + "paragraphs": [ + "The current implementation of Macaw lacks the following actions. We intend to incrementally improve Macaw by supporting more actions and even more advanced techniques for the developed actions.", + "[leftmargin=*]", + "Clarification and Preference Elicitation: Asking clarifying questions has been recently recognized as a necessary component in a conversational system BIBREF3, BIBREF7. The authors are not aware of a published solution for generating clarifying questions using public resources. Therefore, Macaw does not currently support clarification.", + "Explanation: Despite its importance, result list explanation is also a relatively less explored topic. We intend to extend Macaw with result list explanation as soon as we find a stable and mature solution.", + "Recommendation: In our first release, we focus on conversational search and question answering tasks. We intend to provide support for conversational recommendation, e.g., BIBREF13, BIBREF14, BIBREF15, and joint search and recommendation, e.g., BIBREF16, BIBREF17, in the future.", + "Natural Language Interface: Macaw can potentially support access to structured data, such as knowledge graph. We would like to ease conversational natural language interface to structured and semi-structured data in our future releases." + ] + }, + { + "section_name": "Contribution", + "paragraphs": [ + "Macaw is distributed under the MIT License. We welcome contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. This project has adopted the Microsoft Open Source Code of Conduct.", + "When you submit a pull request, a CLA bot will automatically determine whether you need to provide a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions provided by the bot. You will only need to do this once across all repos using our CLA." + ] + }, + { + "section_name": "Conclusions", + "paragraphs": [ + "This paper described Macaw, an open-source platform for conversational information seeking research. Macaw supports multi-turn, multi-modal, and mixed-initiative interactions. It was designed based on a modular architecture that allows further improvements and extensions. Researchers can benefit from Macaw for developing algorithms and techniques for conversational information seeking research, for user studies with different interfaces, for data collection from real users, and for preparing a demonstration of a CIS model." + ] + }, + { + "section_name": "Acknowledgements", + "paragraphs": [ + "The authors wish to thank Ahmed Hassan Awadallah, Krisztian Balog, and Arjen P. de Vries for their invaluable feedback." + ] + } + ], + "qas": [ + { + "question": "Does the paper provide any case studies to illustrate how one can use Macaw for CIS research?", + "question_id": "58ef2442450c392bfc55c4dc35f216542f5f2dbb", + "nlp_background": "infinity", + "topic_background": "unfamiliar", + "paper_read": "no", + "search_query": "information seeking", + "question_writer": "ecca0cede84b7af8a918852311d36346b07f0668", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": false, + "free_form_answer": "", + "evidence": [], + "highlighted_evidence": [] + }, + "annotation_id": "29ef81522bbbf4facade168a4168810a999af0c6", + "worker_id": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": false, + "free_form_answer": "", + "evidence": [], + "highlighted_evidence": [] + }, + "annotation_id": "aab5343f6b18339bce0eff4fd37fa19c43fdaf5b", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + } + ] + }, + { + "question": "What functionality does Macaw provide?", + "question_id": "78a5546e87d4d88e3d9638a0a8cd0b7debf1f09d", + "nlp_background": "infinity", + "topic_background": "unfamiliar", + "paper_read": "no", + "search_query": "information seeking", + "question_writer": "ecca0cede84b7af8a918852311d36346b07f0668", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "Co-Reference Resolution", + "Query Generation", + "Retrieval Model", + "Result Generation" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "Co-Reference Resolution: To support multi-turn interactions, it is sometimes necessary to use co-reference resolution techniques for effective retrieval. In Macaw, we identify all the co-references from the last request of user to the conversation history. The same co-reference resolution outputs can be used for different query generation components. This can be a generic or action-specific component.", + "Query Generation: This component generates a query based on the past user-system interactions. The query generation component may take advantage of co-reference resolution for query expansion or re-writing.", + "Retrieval Model: This is the core ranking component that retrieves documents or passages from a large collection. Macaw can retrieve documents from an arbitrary document collection using the Indri python interface BIBREF9, BIBREF10. We also provide the support for web search using the Bing Web Search API. Macaw also allows multi-stage document re-ranking.", + "Result Generation: The retrieved documents can be too long to be presented using some interfaces. Result generation is basically a post-processing step ran on the retrieved result list. In case of question answering, it can employ answer selection or generation techniques, such as machine reading comprehension models. For example, Macaw features the DrQA model BIBREF11 for question answering." + ], + "highlighted_evidence": [ + "Co-Reference Resolution: To support multi-turn interactions, it is sometimes necessary to use co-reference resolution techniques for effective retrieval. In Macaw, we identify all the co-references from the last request of user to the conversation history. The same co-reference resolution outputs can be used for different query generation components. This can be a generic or action-specific component.\n\nQuery Generation: This component generates a query based on the past user-system interactions. The query generation component may take advantage of co-reference resolution for query expansion or re-writing.\n\nRetrieval Model: This is the core ranking component that retrieves documents or passages from a large collection. Macaw can retrieve documents from an arbitrary document collection using the Indri python interface BIBREF9, BIBREF10. We also provide the support for web search using the Bing Web Search API. Macaw also allows multi-stage document re-ranking.\n\nResult Generation: The retrieved documents can be too long to be presented using some interfaces. Result generation is basically a post-processing step ran on the retrieved result list. In case of question answering, it can employ answer selection or generation techniques, such as machine reading comprehension models. For example, Macaw features the DrQA model BIBREF11 for question answering." + ] + }, + "annotation_id": "dc3777893ff991d532c9322acc44c27ba08f6665", + "worker_id": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "conversational search", + "conversational question answering", + "conversational recommendation", + "conversational natural language interface to structured and semi-structured data" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "Macaw is designed based on a modular architecture to support different information seeking tasks, including conversational search, conversational question answering, conversational recommendation, and conversational natural language interface to structured and semi-structured data. Each interaction in Macaw (from both user and system) is a Message object, thus a conversation is a list of Messages. Macaw consists of multiple actions, each action is a module that can satisfy the information needs of users for some requests. For example, search and question answering can be two actions in Macaw. Even multiple search algorithms can be also seen as multiple actions. Each action can produce multiple outputs (e.g., multiple retrieved documents). For every user interaction, Macaw runs all actions in parallel. The actions' outputs produced within a predefined time interval (i.e., an interaction timeout constant) are then post-processed. Macaw can choose one or combine multiple of these outputs and prepare an output Message object as the user's response." + ], + "highlighted_evidence": [ + "Macaw is designed based on a modular architecture to support different information seeking tasks, including conversational search, conversational question answering, conversational recommendation, and conversational natural language interface to structured and semi-structured data." + ] + }, + "annotation_id": "fc476553658428281b1404241a3ac37b38219284", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + } + ] + }, + { + "question": "What is a wizard of oz setup?", + "question_id": "375b281e7441547ba284068326dd834216e55c07", + "nlp_background": "infinity", + "topic_background": "unfamiliar", + "paper_read": "no", + "search_query": "information seeking", + "question_writer": "ecca0cede84b7af8a918852311d36346b07f0668", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "seeker interacts with a real conversational interface", + "intermediary (or the wizard) receives the seeker's message and performs different information seeking actions" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "Macaw also supports Wizard of Oz studies or intermediary-based information seeking studies. The architecture of Macaw for such setup is presented in FIGREF16. As shown in the figure, the seeker interacts with a real conversational interface that supports multi-modal and mixed-initiative interactions in multiple devices. The intermediary (or the wizard) receives the seeker's message and performs different information seeking actions with Macaw. All seeker-intermediary and intermediary-system interactions will be logged for further analysis. This setup can simulate an ideal CIS system and thus is useful for collecting high-quality data from real users for CIS research." + ], + "highlighted_evidence": [ + "Macaw also supports Wizard of Oz studies or intermediary-based information seeking studies. The architecture of Macaw for such setup is presented in FIGREF16. As shown in the figure, the seeker interacts with a real conversational interface that supports multi-modal and mixed-initiative interactions in multiple devices. The intermediary (or the wizard) receives the seeker's message and performs different information seeking actions with Macaw. All seeker-intermediary and intermediary-system interactions will be logged for further analysis." + ] + }, + "annotation_id": "55d2294e269c04afd444c4993572c0cd2c68d369", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "a setup where the seeker interacts with a real conversational interface and the wizard, an intermediary, performs actions related to the seeker's message", + "evidence": [ + "Macaw also supports Wizard of Oz studies or intermediary-based information seeking studies. The architecture of Macaw for such setup is presented in FIGREF16. As shown in the figure, the seeker interacts with a real conversational interface that supports multi-modal and mixed-initiative interactions in multiple devices. The intermediary (or the wizard) receives the seeker's message and performs different information seeking actions with Macaw. All seeker-intermediary and intermediary-system interactions will be logged for further analysis. This setup can simulate an ideal CIS system and thus is useful for collecting high-quality data from real users for CIS research." + ], + "highlighted_evidence": [ + "Macaw also supports Wizard of Oz studies or intermediary-based information seeking studies. The architecture of Macaw for such setup is presented in FIGREF16. As shown in the figure, the seeker interacts with a real conversational interface that supports multi-modal and mixed-initiative interactions in multiple devices. The intermediary (or the wizard) receives the seeker's message and performs different information seeking actions with Macaw. All seeker-intermediary and intermediary-system interactions will be logged for further analysis. This setup can simulate an ideal CIS system and thus is useful for collecting high-quality data from real users for CIS research." + ] + }, + "annotation_id": "f600861bb2547bd97f257fdd9a0b130ac28e529c", + "worker_id": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4" + } + ] + }, + { + "question": "What interface does Macaw currently have?", + "question_id": "05c49b9f84772e6df41f530d86c1f7a1da6aa489", + "nlp_background": "infinity", + "topic_background": "unfamiliar", + "paper_read": "no", + "search_query": "", + "question_writer": "f7c76ad7ff9c8b54e8c397850358fa59258c6672", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "File IO", + "Standard IO", + "Telegram" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "We have implemented the following interfaces for Macaw:", + "[leftmargin=*]", + "File IO: This interface is designed for experimental purposes, such as evaluating the performance of a conversational search technique on a dataset with multiple queries. This is not an interactive interface.", + "Standard IO: This interactive command line interface is designed for development purposes to interact with the system, see the logs, and debug or improve the system.", + "Telegram: This interactive interface is designed for interaction with real users (see FIGREF4). Telegram is a popular instant messaging service whose client-side code is open-source. We have implemented a Telegram bot that can be used with different devices (personal computers, tablets, and mobile phones) and different operating systems (Android, iOS, Linux, Mac OS, and Windows). This interface allows multi-modal interactions (text, speech, click, image). It can be also used for speech-only interactions. For speech recognition and generation, Macaw relies on online APIs, e.g., the services provided by Google Cloud and Microsoft Azure. In addition, there exist multiple popular groups and channels in Telegram, which allows further integration of social networks with conversational systems. For example, see the Naseri and Zamani's study on news popularity in Telegram BIBREF12." + ], + "highlighted_evidence": [ + "We have implemented the following interfaces for Macaw:\n\n[leftmargin=*]\n\nFile IO: This interface is designed for experimental purposes, such as evaluating the performance of a conversational search technique on a dataset with multiple queries. This is not an interactive interface.\n\nStandard IO: This interactive command line interface is designed for development purposes to interact with the system, see the logs, and debug or improve the system.\n\nTelegram: This interactive interface is designed for interaction with real users (see FIGREF4). Telegram is a popular instant messaging service whose client-side code is open-source. We have implemented a Telegram bot that can be used with different devices (personal computers, tablets, and mobile phones) and different operating systems (Android, iOS, Linux, Mac OS, and Windows). This interface allows multi-modal interactions (text, speech, click, image). It can be also used for speech-only interactions. For speech recognition and generation, Macaw relies on online APIs, e.g., the services provided by Google Cloud and Microsoft Azure. In addition, there exist multiple popular groups and channels in Telegram, which allows further integration of social networks with conversational systems. For example, see the Naseri and Zamani's study on news popularity in Telegram BIBREF12." + ] + }, + "annotation_id": "ddf1457dde6a63a6585ec4f52f4d6ea1984e52eb", + "worker_id": "34c35a1877e453ecaebcf625df3ef788e1953cc4" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "The current implementation of Macaw supports a command line interface as well as mobile, desktop, and web apps." + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "The modular design of Macaw makes it relatively easy to configure a different user interface or add a new one. The current implementation of Macaw supports a command line interface as well as mobile, desktop, and web apps. In more detail, Macaw's interface can be a Telegram bot, which supports a wide range of devices and operating systems (see FIGREF4). This allows Macaw to support multi-modal interactions, such as text, speech, image, click, etc. A number of APIs for automatic speech recognition and generation have been employed to support speech interactions. Note that the Macaw's architecture and implementation allows mixed-initiative interactions." + ], + "highlighted_evidence": [ + "The current implementation of Macaw supports a command line interface as well as mobile, desktop, and web apps." + ] + }, + "annotation_id": "e87df77ac666b446c873ae0e1706b103d36f7d42", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + } + ] + }, + { + "question": "What modalities are supported by Macaw?", + "question_id": "6ecb69360449bb9915ac73c0a816c8ac479cbbfc", + "nlp_background": "infinity", + "topic_background": "unfamiliar", + "paper_read": "no", + "search_query": "", + "question_writer": "f7c76ad7ff9c8b54e8c397850358fa59258c6672", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "text, speech, image, click, etc" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "The modular design of Macaw makes it relatively easy to configure a different user interface or add a new one. The current implementation of Macaw supports a command line interface as well as mobile, desktop, and web apps. In more detail, Macaw's interface can be a Telegram bot, which supports a wide range of devices and operating systems (see FIGREF4). This allows Macaw to support multi-modal interactions, such as text, speech, image, click, etc. A number of APIs for automatic speech recognition and generation have been employed to support speech interactions. Note that the Macaw's architecture and implementation allows mixed-initiative interactions." + ], + "highlighted_evidence": [ + "This allows Macaw to support multi-modal interactions, such as text, speech, image, click, etc." + ] + }, + "annotation_id": "40c72ee3c884a0456fc48639937ed20e6cce5de7", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + } + ] + }, + { + "question": "What are the different modules in Macaw?", + "question_id": "68df324e5fa697baed25c761d0be4c528f7f5cf7", + "nlp_background": "infinity", + "topic_background": "unfamiliar", + "paper_read": "no", + "search_query": "", + "question_writer": "f7c76ad7ff9c8b54e8c397850358fa59258c6672", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "Co-Reference Resolution", + "Query Generation", + "Retrieval Model", + "Result Generation" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "The overview of retrieval and question answering actions in Macaw is shown in FIGREF17. These actions consist of the following components:", + "[leftmargin=*]", + "Co-Reference Resolution: To support multi-turn interactions, it is sometimes necessary to use co-reference resolution techniques for effective retrieval. In Macaw, we identify all the co-references from the last request of user to the conversation history. The same co-reference resolution outputs can be used for different query generation components. This can be a generic or action-specific component.", + "Query Generation: This component generates a query based on the past user-system interactions. The query generation component may take advantage of co-reference resolution for query expansion or re-writing.", + "Retrieval Model: This is the core ranking component that retrieves documents or passages from a large collection. Macaw can retrieve documents from an arbitrary document collection using the Indri python interface BIBREF9, BIBREF10. We also provide the support for web search using the Bing Web Search API. Macaw also allows multi-stage document re-ranking.", + "Result Generation: The retrieved documents can be too long to be presented using some interfaces. Result generation is basically a post-processing step ran on the retrieved result list. In case of question answering, it can employ answer selection or generation techniques, such as machine reading comprehension models. For example, Macaw features the DrQA model BIBREF11 for question answering." + ], + "highlighted_evidence": [ + "These actions consist of the following components:\n\n[leftmargin=*]\n\nCo-Reference Resolution: To support multi-turn interactions, it is sometimes necessary to use co-reference resolution techniques for effective retrieval.", + "Query Generation: This component generates a query based on the past user-system interactions.", + "Retrieval Model: This is the core ranking component that retrieves documents or passages from a large collection.", + "Result Generation: The retrieved documents can be too long to be presented using some interfaces." + ] + }, + "annotation_id": "121c7f194cffbfe8ea3bf64e8197b9ccdc114734", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "Co-Reference Resolution", + "Query Generation", + "Retrieval Model", + "Result Generation" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "Macaw has a modular design, with the goal of making it easy to configure and add new modules such as a different user interface or different retrieval module. The overall setup also follows a Model-View-Controller (MVC) like architecture. The design decisions have been made to smooth the Macaw's adoptions and extensions. Macaw is implemented in Python, thus machine learning models implemented using PyTorch, Scikit-learn, or TensorFlow can be easily integrated into Macaw. The high-level overview of Macaw is depicted in FIGREF8. The user interacts with the interface and the interface produces a Message object from the current interaction of user. The interaction can be in multi-modal form, such as text, speech, image, and click. Macaw stores all interactions in an “Interaction Database”. For every interaction, Macaw looks for most recent user-system interactions (including the system's responses) to create a list of Messages, called the conversation list. It is then dispatched to multiple information seeking (and related) actions. The actions run in parallel, and each should respond within a pre-defined time interval. The output selection component selects from (or potentially combines) the outputs generated by different actions and creates a Message object as the system's response. This message is logged into the interaction database and is sent to the interface to be presented to the user. Again, the response message can be multi-modal and include text, speech, link, list of options, etc.", + "The overview of retrieval and question answering actions in Macaw is shown in FIGREF17. These actions consist of the following components:", + "[leftmargin=*]", + "Co-Reference Resolution: To support multi-turn interactions, it is sometimes necessary to use co-reference resolution techniques for effective retrieval. In Macaw, we identify all the co-references from the last request of user to the conversation history. The same co-reference resolution outputs can be used for different query generation components. This can be a generic or action-specific component.", + "Query Generation: This component generates a query based on the past user-system interactions. The query generation component may take advantage of co-reference resolution for query expansion or re-writing.", + "Retrieval Model: This is the core ranking component that retrieves documents or passages from a large collection. Macaw can retrieve documents from an arbitrary document collection using the Indri python interface BIBREF9, BIBREF10. We also provide the support for web search using the Bing Web Search API. Macaw also allows multi-stage document re-ranking.", + "Result Generation: The retrieved documents can be too long to be presented using some interfaces. Result generation is basically a post-processing step ran on the retrieved result list. In case of question answering, it can employ answer selection or generation techniques, such as machine reading comprehension models. For example, Macaw features the DrQA model BIBREF11 for question answering." + ], + "highlighted_evidence": [ + "Macaw has a modular design, with the goal of making it easy to configure and add new modules such as a different user interface or different retrieval module. The overall setup also follows a Model-View-Controller (MVC) like architecture.", + "These actions consist of the following components:\n\n[leftmargin=*]\n\nCo-Reference Resolution: To support multi-turn interactions, it is sometimes necessary to use co-reference resolution techniques for effective retrieval. In Macaw, we identify all the co-references from the last request of user to the conversation history. The same co-reference resolution outputs can be used for different query generation components. This can be a generic or action-specific component.\n\nQuery Generation: This component generates a query based on the past user-system interactions. The query generation component may take advantage of co-reference resolution for query expansion or re-writing.\n\nRetrieval Model: This is the core ranking component that retrieves documents or passages from a large collection. Macaw can retrieve documents from an arbitrary document collection using the Indri python interface BIBREF9, BIBREF10. We also provide the support for web search using the Bing Web Search API. Macaw also allows multi-stage document re-ranking.\n\nResult Generation: The retrieved documents can be too long to be presented using some interfaces. Result generation is basically a post-processing step ran on the retrieved result list. In case of question answering, it can employ answer selection or generation techniques, such as machine reading comprehension models. For example, Macaw features the DrQA model BIBREF11 for question answering." + ] + }, + "annotation_id": "e0c9834bf81d5a27133d8331ad87c42b03b3be74", + "worker_id": "34c35a1877e453ecaebcf625df3ef788e1953cc4" + } + ] + } + ], + "figures_and_tables": [ + { + "file": "1-Figure1-1.png", + "caption": "Figure 1: Example screenshots of the Macaw interface on mobile devices using Telegram bots. Macaw supports multimodal and multi-turn interactions." + }, + { + "file": "3-Figure2-1.png", + "caption": "Figure 2: The high-level architecture of Macaw for developing conversation information seeking systems." + }, + { + "file": "3-Figure3-1.png", + "caption": "Figure 3: The high-level architecture of Macaw for user studies. In this architecture, user interacts with a human intermediary who is an expert of the system and can interact with the system to address the user’s information need." + }, + { + "file": "4-Figure4-1.png", + "caption": "Figure 4: The overview of retrieval and question answering in Macaw." + } + ] + }, + "1611.03599": { + "title": "UTCNN: a Deep Learning Model of Stance Classificationon on Social Media Text", + "abstract": "Most neural network models for document classification on social media focus on text infor-mation to the neglect of other information on these platforms. In this paper, we classify post stance on social media channels and develop UTCNN, a neural network model that incorporates user tastes, topic tastes, and user comments on posts. UTCNN not only works on social media texts, but also analyzes texts in forums and message boards. Experiments performed on Chinese Facebook data and English online debate forum data show that UTCNN achieves a 0.755 macro-average f-score for supportive, neutral, and unsupportive stance classes on Facebook data, which is significantly better than models in which either user, topic, or comment information is withheld. This model design greatly mitigates the lack of data for the minor class without the use of oversampling. In addition, UTCNN yields a 0.842 accuracy on English online debate forum data, which also significantly outperforms results from previous work as well as other deep learning models, showing that UTCNN performs well regardless of language or platform.", + "full_text": [ + { + "section_name": "Introduction", + "paragraphs": [ + " This work is licenced under a Creative Commons Attribution 4.0 International License. License details: http://creativecommons.org/licenses/by/4.0/ Deep neural networks have been widely used in text classification and have achieved promising results BIBREF0 , BIBREF1 , BIBREF2 . Most focus on content information and use models such as convolutional neural networks (CNN) BIBREF3 or recursive neural networks BIBREF4 . However, for user-generated posts on social media like Facebook or Twitter, there is more information that should not be ignored. On social media platforms, a user can act either as the author of a post or as a reader who expresses his or her comments about the post.", + "In this paper, we classify posts taking into account post authorship, likes, topics, and comments. In particular, users and their “likes” hold strong potential for text mining. For example, given a set of posts that are related to a specific topic, a user's likes and dislikes provide clues for stance labeling. From a user point of view, users with positive attitudes toward the issue leave positive comments on the posts with praise or even just the post's content; from a post point of view, positive posts attract users who hold positive stances. We also investigate the influence of topics: different topics are associated with different stance labeling tendencies and word usage. For example we discuss women's rights and unwanted babies on the topic of abortion, but we criticize medicine usage or crime when on the topic of marijuana BIBREF5 . Even for posts on a specific topic like nuclear power, a variety of arguments are raised: green energy, radiation, air pollution, and so on. As for comments, we treat them as additional text information. The arguments in the comments and the commenters (the users who leave the comments) provide hints on the post's content and further facilitate stance classification.", + "In this paper, we propose the user-topic-comment neural network (UTCNN), a deep learning model that utilizes user, topic, and comment information. We attempt to learn user and topic representations which encode user interactions and topic influences to further enhance text classification, and we also incorporate comment information. We evaluate this model on a post stance classification task on forum-style social media platforms. The contributions of this paper are as follows: 1. We propose UTCNN, a neural network for text in modern social media channels as well as legacy social media, forums, and message boards — anywhere that reveals users, their tastes, as well as their replies to posts. 2. When classifying social media post stances, we leverage users, including authors and likers. User embeddings can be generated even for users who have never posted anything. 3. We incorporate a topic model to automatically assign topics to each post in a single topic dataset. 4. We show that overall, the proposed method achieves the highest performance in all instances, and that all of the information extracted, whether users, topics, or comments, still has its contributions." + ] + }, + { + "section_name": "Extra-Linguistic Features for Stance Classification", + "paragraphs": [ + "In this paper we aim to use text as well as other features to see how they complement each other in a deep learning model. In the stance classification domain, previous work has showed that text features are limited, suggesting that adding extra-linguistic constraints could improve performance BIBREF6 , BIBREF7 , BIBREF8 . For example, Hasan and Ng as well as Thomas et al. require that posts written by the same author have the same stance BIBREF9 , BIBREF10 . The addition of this constraint yields accuracy improvements of 1–7% for some models and datasets. Hasan and Ng later added user-interaction constraints and ideology constraints BIBREF7 : the former models the relationship among posts in a sequence of replies and the latter models inter-topic relationships, e.g., users who oppose abortion could be conservative and thus are likely to oppose gay rights.", + "For work focusing on online forum text, since posts are linked through user replies, sequential labeling methods have been used to model relationships between posts. For example, Hasan and Ng use hidden Markov models (HMMs) to model dependent relationships to the preceding post BIBREF9 ; Burfoot et al. use iterative classification to repeatedly generate new estimates based on the current state of knowledge BIBREF11 ; Sridhar et al. use probabilistic soft logic (PSL) to model reply links via collaborative filtering BIBREF12 . In the Facebook dataset we study, we use comments instead of reply links. However, as the ultimate goal in this paper is predicting not comment stance but post stance, we treat comments as extra information for use in predicting post stance." + ] + }, + { + "section_name": "Deep Learning on Extra-Linguistic Features", + "paragraphs": [ + "In recent years neural network models have been applied to document sentiment classification BIBREF13 , BIBREF4 , BIBREF14 , BIBREF15 , BIBREF2 . Text features can be used in deep networks to capture text semantics or sentiment. For example, Dong et al. use an adaptive layer in a recursive neural network for target-dependent Twitter sentiment analysis, where targets are topics such as windows 7 or taylor swift BIBREF16 , BIBREF17 ; recursive neural tensor networks (RNTNs) utilize sentence parse trees to capture sentence-level sentiment for movie reviews BIBREF4 ; Le and Mikolov predict sentiment by using paragraph vectors to model each paragraph as a continuous representation BIBREF18 . They show that performance can thus be improved by more delicate text models.", + "Others have suggested using extra-linguistic features to improve the deep learning model. The user-word composition vector model (UWCVM) BIBREF19 is inspired by the possibility that the strength of sentiment words is user-specific; to capture this they add user embeddings in their model. In UPNN, a later extension, they further add a product-word composition as product embeddings, arguing that products can also show different tendencies of being rated or reviewed BIBREF20 . Their addition of user information yielded 2–10% improvements in accuracy as compared to the above-mentioned RNTN and paragraph vector methods. We also seek to inject user information into the neural network model. In comparison to the research of Tang et al. on sentiment classification for product reviews, the difference is two-fold. First, we take into account multiple users (one author and potentially many likers) for one post, whereas only one user (the reviewer) is involved in a review. Second, we add comment information to provide more features for post stance classification. None of these two factors have been considered previously in a deep learning model for text stance classification. Therefore, we propose UTCNN, which generates and utilizes user embeddings for all users — even for those who have not authored any posts — and incorporates comments to further improve performance." + ] + }, + { + "section_name": "Method", + "paragraphs": [ + "In this section, we first describe CNN-based document composition, which captures user- and topic-dependent document-level semantic representation from word representations. Then we show how to add comment information to construct the user-topic-comment neural network (UTCNN)." + ] + }, + { + "section_name": "User- and Topic-dependent Document Composition", + "paragraphs": [ + "As shown in Figure FIGREF4 , we use a general CNN BIBREF3 and two semantic transformations for document composition . We are given a document with an engaged user INLINEFORM0 , a topic INLINEFORM1 , and its composite INLINEFORM2 words, each word INLINEFORM3 of which is associated with a word embedding INLINEFORM4 where INLINEFORM5 is the vector dimension. For each word embedding INLINEFORM6 , we apply two dot operations as shown in Equation EQREF6 : DISPLAYFORM0 ", + "where INLINEFORM0 models the user reading preference for certain semantics, and INLINEFORM1 models the topic semantics; INLINEFORM2 and INLINEFORM3 are the dimensions of transformed user and topic embeddings respectively. We use INLINEFORM4 to model semantically what each user prefers to read and/or write, and use INLINEFORM5 to model the semantics of each topic. The dot operation of INLINEFORM6 and INLINEFORM7 transforms the global representation INLINEFORM8 to a user-dependent representation. Likewise, the dot operation of INLINEFORM9 and INLINEFORM10 transforms INLINEFORM11 to a topic-dependent representation.", + "After the two dot operations on INLINEFORM0 , we have user-dependent and topic-dependent word vectors INLINEFORM1 and INLINEFORM2 , which are concatenated to form a user- and topic-dependent word vector INLINEFORM3 . Then the transformed word embeddings INLINEFORM4 are used as the CNN input. Here we apply three convolutional layers on the concatenated transformed word embeddings INLINEFORM5 : DISPLAYFORM0 ", + "where INLINEFORM0 is the index of words; INLINEFORM1 is a non-linear activation function (we use INLINEFORM2 ); INLINEFORM5 is the convolutional filter with input length INLINEFORM6 and output length INLINEFORM7 , where INLINEFORM8 is the window size of the convolutional operation; and INLINEFORM9 and INLINEFORM10 are the output and bias of the convolution layer INLINEFORM11 , respectively. In our experiments, the three window sizes INLINEFORM12 in the three convolution layers are one, two, and three, encoding unigram, bigram, and trigram semantics accordingly.", + "After the convolutional layer, we add a maximum pooling layer among convolutional outputs to obtain the unigram, bigram, and trigram n-gram representations. This is succeeded by an average pooling layer for an element-wise average of the three maximized convolution outputs." + ] + }, + { + "section_name": "UTCNN Model Description", + "paragraphs": [ + "Figure FIGREF10 illustrates the UTCNN model. As more than one user may interact with a given post, we first add a maximum pooling layer after the user matrix embedding layer and user vector embedding layer to form a moderator matrix embedding INLINEFORM0 and a moderator vector embedding INLINEFORM1 for moderator INLINEFORM2 respectively, where INLINEFORM3 is used for the semantic transformation in the document composition process, as mentioned in the previous section. The term moderator here is to denote the pseudo user who provides the overall semantic/sentiment of all the engaged users for one document. The embedding INLINEFORM4 models the moderator stance preference, that is, the pattern of the revealed user stance: whether a user is willing to show his preference, whether a user likes to show impartiality with neutral statements and reasonable arguments, or just wants to show strong support for one stance. Ideally, the latent user stance is modeled by INLINEFORM5 for each user. Likewise, for topic information, a maximum pooling layer is added after the topic matrix embedding layer and topic vector embedding layer to form a joint topic matrix embedding INLINEFORM6 and a joint topic vector embedding INLINEFORM7 for topic INLINEFORM8 respectively, where INLINEFORM9 models the semantic transformation of topic INLINEFORM10 as in users and INLINEFORM11 models the topic stance tendency. The latent topic stance is also modeled by INLINEFORM12 for each topic.", + "As for comments, we view them as short documents with authors only but without likers nor their own comments. Therefore we apply document composition on comments although here users are commenters (users who comment). It is noticed that the word embeddings INLINEFORM0 for the same word in the posts and comments are the same, but after being transformed to INLINEFORM1 in the document composition process shown in Figure FIGREF4 , they might become different because of their different engaged users. The output comment representation together with the commenter vector embedding INLINEFORM2 and topic vector embedding INLINEFORM3 are concatenated and a maximum pooling layer is added to select the most important feature for comments. Instead of requiring that the comment stance agree with the post, UTCNN simply extracts the most important features of the comment contents; they could be helpful, whether they show obvious agreement or disagreement. Therefore when combining comment information here, the maximum pooling layer is more appropriate than other pooling or merging layers. Indeed, we believe this is one reason for UTCNN's performance gains.", + "Finally, the pooled comment representation, together with user vector embedding INLINEFORM0 , topic vector embedding INLINEFORM1 , and document representation are fed to a fully connected network, and softmax is applied to yield the final stance label prediction for the post." + ] + }, + { + "section_name": "Experiment", + "paragraphs": [ + "We start with the experimental dataset and then describe the training process as well as the implementation of the baselines. We also implement several variations to reveal the effects of features: authors, likers, comment, and commenters. In the results section we compare our model with related work." + ] + }, + { + "section_name": "Dataset", + "paragraphs": [ + "We tested the proposed UTCNN on two different datasets: FBFans and CreateDebate. FBFans is a privately-owned, single-topic, Chinese, unbalanced, social media dataset, and CreateDebate is a public, multiple-topic, English, balanced, forum dataset. Results using these two datasets show the applicability and superiority for different topics, languages, data distributions, and platforms.", + "The FBFans dataset contains data from anti-nuclear-power Chinese Facebook fan groups from September 2013 to August 2014, including posts and their author and liker IDs. There are a total of 2,496 authors, 505,137 likers, 33,686 commenters, and 505,412 unique users. Two annotators were asked to take into account only the post content to label the stance of the posts in the whole dataset as supportive, neutral, or unsupportive (hereafter denoted as Sup, Neu, and Uns). Sup/Uns posts were those in support of or against anti-reconstruction; Neu posts were those evincing a neutral standpoint on the topic, or were irrelevant. Raw agreement between annotators is 0.91, indicating high agreement. Specifically, Cohen’s Kappa for Neu and not Neu labeling is 0.58 (moderate), and for Sup or Uns labeling is 0.84 (almost perfect). Posts with inconsistent labels were filtered out, and the development and testing sets were randomly selected from what was left. Posts in the development and testing sets involved at least one user who appeared in the training set. The number of posts for each stance is shown on the left-hand side of Table TABREF12 . About twenty percent of the posts were labeled with a stance, and the number of supportive (Sup) posts was much larger than that of the unsupportive (Uns) ones: this is thus highly skewed data, which complicates stance classification. On average, 161.1 users were involved in one post. The maximum was 23,297 and the minimum was one (the author). For comments, on average there were 3 comments per post. The maximum was 1,092 and the minimum was zero.", + "To test whether the assumption of this paper – posts attract users who hold the same stance to like them – is reliable, we examine the likes from authors of different stances. Posts in FBFans dataset are used for this analysis. We calculate the like statistics of each distinct author from these 32,595 posts. As the numbers of authors in the Sup, Neu and Uns stances are largely imbalanced, these numbers are normalized by the number of users of each stance. Table TABREF13 shows the results. Posts with stances (i.e., not neutral) attract users of the same stance. Neutral posts also attract both supportive and neutral users, like what we observe in supportive posts, but just the neutral posts can attract even more neutral likers. These results do suggest that users prefer posts of the same stance, or at least posts of no obvious stance which might cause annoyance when reading, and hence support the user modeling in our approach.", + "The CreateDebate dataset was collected from an English online debate forum discussing four topics: abortion (ABO), gay rights (GAY), Obama (OBA), and marijuana (MAR). The posts are annotated as for (F) and against (A). Replies to posts in this dataset are also labeled with stance and hence use the same data format as posts. The labeling results are shown in the right-hand side of Table TABREF12 . We observe that the dataset is more balanced than the FBFans dataset. In addition, there are 977 unique users in the dataset. To compare with Hasan and Ng's work, we conducted five-fold cross-validation and present the annotation results as the average number of all folds BIBREF9 , BIBREF5 .", + "The FBFans dataset has more integrated functions than the CreateDebate dataset; thus our model can utilize all linguistic and extra-linguistic features. For the CreateDebate dataset, on the other hand, the like and comment features are not available (as there is a stance label for each reply, replies are evaluated as posts as other previous work) but we still implemented our model using the content, author, and topic information." + ] + }, + { + "section_name": "Settings", + "paragraphs": [ + "In the UTCNN training process, cross-entropy was used as the loss function and AdaGrad as the optimizer. For FBFans dataset, we learned the 50-dimensional word embeddings on the whole dataset using GloVe BIBREF21 to capture the word semantics; for CreateDebate dataset we used the publicly available English 50-dimensional word embeddings, pre-trained also using GloVe. These word embeddings were fixed in the training process. The learning rate was set to 0.03. All user and topic embeddings were randomly initialized in the range of [-0.1 0.1]. Matrix embeddings for users and topics were sized at 250 ( INLINEFORM0 ); vector embeddings for users and topics were set to length 10.", + "We applied the LDA topic model BIBREF22 on the FBFans dataset to determine the latent topics with which to build topic embeddings, as there is only one general known topic: nuclear power plants. We learned 100 latent topics and assigned the top three topics for each post. For the CreateDebate dataset, which itself constitutes four topics, the topic labels for posts were used directly without additionally applying LDA.", + "For the FBFans data we report class-based f-scores as well as the macro-average f-score ( INLINEFORM0 ) shown in equation EQREF19 . DISPLAYFORM0 ", + "where INLINEFORM0 and INLINEFORM1 are the average precision and recall of the three class. We adopted the macro-average f-score as the evaluation metric for the overall performance because (1) the experimental dataset is severely imbalanced, which is common for contentious issues; and (2) for stance classification, content in minor-class posts is usually more important for further applications. For the CreateDebate dataset, accuracy was adopted as the evaluation metric to compare the results with related work BIBREF7 , BIBREF9 , BIBREF12 ." + ] + }, + { + "section_name": "Baselines", + "paragraphs": [ + "We pit our model against the following baselines: 1) SVM with unigram, bigram, and trigram features, which is a standard yet rather strong classifier for text features; 2) SVM with average word embedding, where a document is represented as a continuous representation by averaging the embeddings of the composite words; 3) SVM with average transformed word embeddings (the INLINEFORM0 in equation EQREF6 ), where a document is represented as a continuous representation by averaging the transformed embeddings of the composite words; 4) two mature deep learning models on text classification, CNN BIBREF3 and Recurrent Convolutional Neural Networks (RCNN) BIBREF0 , where the hyperparameters are based on their work; 5) the above SVM and deep learning models with comment information; 6) UTCNN without user information, representing a pure-text CNN model where we use the same user matrix and user embeddings INLINEFORM1 and INLINEFORM2 for each user; 7) UTCNN without the LDA model, representing how UTCNN works with a single-topic dataset; 8) UTCNN without comments, in which the model predicts the stance label given only user and topic information. All these models were trained on the training set, and parameters as well as the SVM kernel selections (linear or RBF) were fine-tuned on the development set. Also, we adopt oversampling on SVMs, CNN and RCNN because the FBFans dataset is highly imbalanced." + ] + }, + { + "section_name": "Results on FBFans Dataset", + "paragraphs": [ + "In Table TABREF22 we show the results of UTCNN and the baselines on the FBFans dataset. Here Majority yields good performance on Neu since FBFans is highly biased to the neutral class. The SVM models perform well on Sup and Neu but perform poorly for Uns, showing that content information in itself is insufficient to predict stance labels, especially for the minor class. With the transformed word embedding feature, SVM can achieve comparable performance as SVM with n-gram feature. However, the much fewer feature dimension of the transformed word embedding makes SVM with word embeddings a more efficient choice for modeling the large scale social media dataset. For the CNN and RCNN models, they perform slightly better than most of the SVM models but still, the content information is insufficient to achieve a good performance on the Uns posts. As to adding comment information to these models, since the commenters do not always hold the same stance as the author, simply adding comments and post contents together merely adds noise to the model.", + "Among all UTCNN variations, we find that user information is most important, followed by topic and comment information. UTCNN without user information shows results similar to SVMs — it does well for Sup and Neu but detects no Uns. Its best f-scores on both Sup and Neu among all methods show that with enough training data, content-based models can perform well; at the same time, the lack of user information results in too few clues for minor-class posts to either predict their stance directly or link them to other users and posts for improved performance. The 17.5% improvement when adding user information suggests that user information is especially useful when the dataset is highly imbalanced. All models that consider user information predict the minority class successfully. UCTNN without topic information works well but achieves lower performance than the full UTCNN model. The 4.9% performance gain brought by LDA shows that although it is satisfactory for single topic datasets, adding that latent topics still benefits performance: even when we are discussing the same topic, we use different arguments and supporting evidence. Lastly, we get 4.8% improvement when adding comment information and it achieves comparable performance to UTCNN without topic information, which shows that comments also benefit performance. For platforms where user IDs are pixelated or otherwise hidden, adding comments to a text model still improves performance. In its integration of user, content, and comment information, the full UTCNN produces the highest f-scores on all Sup, Neu, and Uns stances among models that predict the Uns class, and the highest macro-average f-score overall. This shows its ability to balance a biased dataset and supports our claim that UTCNN successfully bridges content and user, topic, and comment information for stance classification on social media text. Another merit of UTCNN is that it does not require a balanced training data. This is supported by its outperforming other models though no oversampling technique is applied to the UTCNN related experiments as shown in this paper. Thus we can conclude that the user information provides strong clues and it is still rich even in the minority class.", + "We also investigate the semantic difference when a user acts as an author/liker or a commenter. We evaluated a variation in which all embeddings from the same user were forced to be identical (this is the UTCNN shared user embedding setting in Table TABREF22 ). This setting yielded only a 2.5% improvement over the model without comments, which is not statistically significant. However, when separating authors/likers and commenters embeddings (i.e., the UTCNN full model), we achieved much greater improvements (4.8%). We attribute this result to the tendency of users to use different wording for different roles (for instance author vs commenter). This is observed when the user, acting as an author, attempts to support her argument against nuclear power by using improvements in solar power; when acting as a commenter, though, she interacts with post contents by criticizing past politicians who supported nuclear power or by arguing that the proposed evacuation plan in case of a nuclear accident is ridiculous. Based on this finding, in the final UTCNN setting we train two user matrix embeddings for one user: one for the author/liker role and the other for the commenter role." + ] + }, + { + "section_name": "Results on CreateDebate Dataset", + "paragraphs": [ + "Table TABREF24 shows the results of UTCNN, baselines as we implemented on the FBFans datset and related work on the CreateDebate dataset. We do not adopt oversampling on these models because the CreateDebate dataset is almost balanced. In previous work, integer linear programming (ILP) or linear-chain conditional random fields (CRFs) were proposed to integrate text features, author, ideology, and user-interaction constraints, where text features are unigram, bigram, and POS-dependencies; the author constraint tends to require that posts from the same author for the same topic hold the same stance; the ideology constraint aims to capture inferences between topics for the same author; the user-interaction constraint models relationships among posts via user interactions such as replies BIBREF7 , BIBREF9 .", + "The SVM with n-gram or average word embedding feature performs just similar to the majority. However, with the transformed word embedding, it achieves superior results. It shows that the learned user and topic embeddings really capture the user and topic semantics. This finding is not so obvious in the FBFans dataset and it might be due to the unfavorable data skewness for SVM. As for CNN and RCNN, they perform slightly better than most SVMs as we found in Table TABREF22 for FBFans.", + "Compared to the ILP BIBREF7 and CRF BIBREF9 methods, the UTCNN user embeddings encode author and user-interaction constraints, where the ideology constraint is modeled by the topic embeddings and text features are modeled by the CNN. The significant improvement achieved by UTCNN suggests the latent representations are more effective than overt model constraints.", + "The PSL model BIBREF12 jointly labels both author and post stance using probabilistic soft logic (PSL) BIBREF23 by considering text features and reply links between authors and posts as in Hasan and Ng's work. Table TABREF24 reports the result of their best AD setting, which represents the full joint stance/disagreement collective model on posts and is hence more relevant to UTCNN. In contrast to their model, the UTCNN user embeddings represent relationships between authors, but UTCNN models do not utilize link information between posts. Though the PSL model has the advantage of being able to jointly label the stances of authors and posts, its performance on posts is lower than the that for the ILP or CRF models. UTCNN significantly outperforms these models on posts and has the potential to predict user stances through the generated user embeddings.", + "For the CreateDebate dataset, we also evaluated performance when not using topic embeddings or user embeddings; as replies in this dataset are viewed as posts, the setting without comment embeddings is not available. Table TABREF24 shows the same findings as Table TABREF22 : the 21% improvement in accuracy demonstrates that user information is the most vital. This finding also supports the results in the related work: user constraints are useful and can yield 11.2% improvement in accuracy BIBREF7 . Further considering topic information yields 3.4% improvement, suggesting that knowing the subject of debates provides useful information. In sum, Table TABREF22 together with Table TABREF24 show that UTCNN achieves promising performance regardless of topic, language, data distribution, and platform." + ] + }, + { + "section_name": "Conclusion", + "paragraphs": [ + "We have proposed UTCNN, a neural network model that incorporates user, topic, content and comment information for stance classification on social media texts. UTCNN learns user embeddings for all users with minimum active degree, i.e., one post or one like. Topic information obtained from the topic model or the pre-defined labels further improves the UTCNN model. In addition, comment information provides additional clues for stance classification. We have shown that UTCNN achieves promising and balanced results. In the future we plan to explore the effectiveness of the UTCNN user embeddings for author stance classification." + ] + }, + { + "section_name": "Acknowledgements", + "paragraphs": [ + "Research of this paper was partially supported by Ministry of Science and Technology, Taiwan, under the contract MOST 104-2221-E-001-024-MY2." + ] + } + ], + "qas": [ + { + "question": "What topic is covered in the Chinese Facebook data? ", + "question_id": "37a79be0148e1751ffb2daabe4c8ec6680036106", + "nlp_background": "", + "topic_background": "", + "paper_read": "", + "search_query": "social media", + "question_writer": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "anti-nuclear-power" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "The FBFans dataset contains data from anti-nuclear-power Chinese Facebook fan groups from September 2013 to August 2014, including posts and their author and liker IDs. There are a total of 2,496 authors, 505,137 likers, 33,686 commenters, and 505,412 unique users. Two annotators were asked to take into account only the post content to label the stance of the posts in the whole dataset as supportive, neutral, or unsupportive (hereafter denoted as Sup, Neu, and Uns). Sup/Uns posts were those in support of or against anti-reconstruction; Neu posts were those evincing a neutral standpoint on the topic, or were irrelevant. Raw agreement between annotators is 0.91, indicating high agreement. Specifically, Cohen’s Kappa for Neu and not Neu labeling is 0.58 (moderate), and for Sup or Uns labeling is 0.84 (almost perfect). Posts with inconsistent labels were filtered out, and the development and testing sets were randomly selected from what was left. Posts in the development and testing sets involved at least one user who appeared in the training set. The number of posts for each stance is shown on the left-hand side of Table TABREF12 . About twenty percent of the posts were labeled with a stance, and the number of supportive (Sup) posts was much larger than that of the unsupportive (Uns) ones: this is thus highly skewed data, which complicates stance classification. On average, 161.1 users were involved in one post. The maximum was 23,297 and the minimum was one (the author). For comments, on average there were 3 comments per post. The maximum was 1,092 and the minimum was zero." + ], + "highlighted_evidence": [ + "The FBFans dataset contains data from anti-nuclear-power Chinese Facebook fan groups from September 2013 to August 2014, including posts and their author and liker IDs." + ] + }, + "annotation_id": "08c093860f115e2b178c70128098ea69a1430ab8", + "worker_id": "c1018a31c3272ce74964a3280069f62f314a1a58" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "anti-nuclear-power" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "The FBFans dataset contains data from anti-nuclear-power Chinese Facebook fan groups from September 2013 to August 2014, including posts and their author and liker IDs. There are a total of 2,496 authors, 505,137 likers, 33,686 commenters, and 505,412 unique users. Two annotators were asked to take into account only the post content to label the stance of the posts in the whole dataset as supportive, neutral, or unsupportive (hereafter denoted as Sup, Neu, and Uns). Sup/Uns posts were those in support of or against anti-reconstruction; Neu posts were those evincing a neutral standpoint on the topic, or were irrelevant. Raw agreement between annotators is 0.91, indicating high agreement. Specifically, Cohen’s Kappa for Neu and not Neu labeling is 0.58 (moderate), and for Sup or Uns labeling is 0.84 (almost perfect). Posts with inconsistent labels were filtered out, and the development and testing sets were randomly selected from what was left. Posts in the development and testing sets involved at least one user who appeared in the training set. The number of posts for each stance is shown on the left-hand side of Table TABREF12 . About twenty percent of the posts were labeled with a stance, and the number of supportive (Sup) posts was much larger than that of the unsupportive (Uns) ones: this is thus highly skewed data, which complicates stance classification. On average, 161.1 users were involved in one post. The maximum was 23,297 and the minimum was one (the author). For comments, on average there were 3 comments per post. The maximum was 1,092 and the minimum was zero." + ], + "highlighted_evidence": [ + "The FBFans dataset contains data from anti-nuclear-power Chinese Facebook fan groups from September 2013 to August 2014, including posts and their author and liker IDs. There are a total of 2,496 authors, 505,137 likers, 33,686 commenters, and 505,412 unique users. Two annotators were asked to take into account only the post content to label the stance of the posts in the whole dataset as supportive, neutral, or unsupportive (hereafter denoted as Sup, Neu, and Uns). Sup/Uns posts were those in support of or against anti-reconstruction; Neu posts were those evincing a neutral standpoint on the topic, or were irrelevant. " + ] + }, + "annotation_id": "78a382b17b1c41f970eb19f3cdf32c8750f2c46e", + "worker_id": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4" + } + ] + }, + { + "question": "How many layers does the UTCNN model have?", + "question_id": "518dae6f936882152c162058895db4eca815e649", + "nlp_background": "", + "topic_background": "", + "paper_read": "", + "search_query": "social media", + "question_writer": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "eight layers", + "evidence": [ + "Figure FIGREF10 illustrates the UTCNN model. As more than one user may interact with a given post, we first add a maximum pooling layer after the user matrix embedding layer and user vector embedding layer to form a moderator matrix embedding INLINEFORM0 and a moderator vector embedding INLINEFORM1 for moderator INLINEFORM2 respectively, where INLINEFORM3 is used for the semantic transformation in the document composition process, as mentioned in the previous section. The term moderator here is to denote the pseudo user who provides the overall semantic/sentiment of all the engaged users for one document. The embedding INLINEFORM4 models the moderator stance preference, that is, the pattern of the revealed user stance: whether a user is willing to show his preference, whether a user likes to show impartiality with neutral statements and reasonable arguments, or just wants to show strong support for one stance. Ideally, the latent user stance is modeled by INLINEFORM5 for each user. Likewise, for topic information, a maximum pooling layer is added after the topic matrix embedding layer and topic vector embedding layer to form a joint topic matrix embedding INLINEFORM6 and a joint topic vector embedding INLINEFORM7 for topic INLINEFORM8 respectively, where INLINEFORM9 models the semantic transformation of topic INLINEFORM10 as in users and INLINEFORM11 models the topic stance tendency. The latent topic stance is also modeled by INLINEFORM12 for each topic.", + "As for comments, we view them as short documents with authors only but without likers nor their own comments. Therefore we apply document composition on comments although here users are commenters (users who comment). It is noticed that the word embeddings INLINEFORM0 for the same word in the posts and comments are the same, but after being transformed to INLINEFORM1 in the document composition process shown in Figure FIGREF4 , they might become different because of their different engaged users. The output comment representation together with the commenter vector embedding INLINEFORM2 and topic vector embedding INLINEFORM3 are concatenated and a maximum pooling layer is added to select the most important feature for comments. Instead of requiring that the comment stance agree with the post, UTCNN simply extracts the most important features of the comment contents; they could be helpful, whether they show obvious agreement or disagreement. Therefore when combining comment information here, the maximum pooling layer is more appropriate than other pooling or merging layers. Indeed, we believe this is one reason for UTCNN's performance gains.", + "Finally, the pooled comment representation, together with user vector embedding INLINEFORM0 , topic vector embedding INLINEFORM1 , and document representation are fed to a fully connected network, and softmax is applied to yield the final stance label prediction for the post." + ], + "highlighted_evidence": [ + "Figure FIGREF10 illustrates the UTCNN model. As more than one user may interact with a given post, we first add a maximum pooling layer after the user matrix embedding layer and user vector embedding layer to form a moderator matrix embedding INLINEFORM0 and a moderator vector embedding INLINEFORM1 for moderator INLINEFORM2 respectively, where INLINEFORM3 is used for the semantic transformation in the document composition process, as mentioned in the previous section. The term moderator here is to denote the pseudo user who provides the overall semantic/sentiment of all the engaged users for one document. The embedding INLINEFORM4 models the moderator stance preference, that is, the pattern of the revealed user stance: whether a user is willing to show his preference, whether a user likes to show impartiality with neutral statements and reasonable arguments, or just wants to show strong support for one stance. Ideally, the latent user stance is modeled by INLINEFORM5 for each user. Likewise, for topic information, a maximum pooling layer is added after the topic matrix embedding layer and topic vector embedding layer to form a joint topic matrix embedding INLINEFORM6 and a joint topic vector embedding INLINEFORM7 for topic INLINEFORM8 respectively, where INLINEFORM9 models the semantic transformation of topic INLINEFORM10 as in users and INLINEFORM11 models the topic stance tendency. The latent topic stance is also modeled by INLINEFORM12 for each topic.\n\nAs for comments, we view them as short documents with authors only but without likers nor their own comments. Therefore we apply document composition on comments although here users are commenters (users who comment). It is noticed that the word embeddings INLINEFORM0 for the same word in the posts and comments are the same, but after being transformed to INLINEFORM1 in the document composition process shown in Figure FIGREF4 , they might become different because of their different engaged users. The output comment representation together with the commenter vector embedding INLINEFORM2 and topic vector embedding INLINEFORM3 are concatenated and a maximum pooling layer is added to select the most important feature for comments. Instead of requiring that the comment stance agree with the post, UTCNN simply extracts the most important features of the comment contents; they could be helpful, whether they show obvious agreement or disagreement. Therefore when combining comment information here, the maximum pooling layer is more appropriate than other pooling or merging layers. Indeed, we believe this is one reason for UTCNN's performance gains.\n\nFinally, the pooled comment representation, together with user vector embedding INLINEFORM0 , topic vector embedding INLINEFORM1 , and document representation are fed to a fully connected network, and softmax is applied to yield the final stance label prediction for the post." + ] + }, + "annotation_id": "87ce7f55c6ac51ba04c9a584ca7de029f0496d87", + "worker_id": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4" + } + ] + }, + { + "question": "What topics are included in the debate data?", + "question_id": "e44a6bf67ce3fde0c6608b150030e44d87eb25e3", + "nlp_background": "", + "topic_background": "", + "paper_read": "", + "search_query": "social media", + "question_writer": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "abortion", + "gay rights", + "Obama", + "marijuana" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "The CreateDebate dataset was collected from an English online debate forum discussing four topics: abortion (ABO), gay rights (GAY), Obama (OBA), and marijuana (MAR). The posts are annotated as for (F) and against (A). Replies to posts in this dataset are also labeled with stance and hence use the same data format as posts. The labeling results are shown in the right-hand side of Table TABREF12 . We observe that the dataset is more balanced than the FBFans dataset. In addition, there are 977 unique users in the dataset. To compare with Hasan and Ng's work, we conducted five-fold cross-validation and present the annotation results as the average number of all folds BIBREF9 , BIBREF5 ." + ], + "highlighted_evidence": [ + "The CreateDebate dataset was collected from an English online debate forum discussing four topics: abortion (ABO), gay rights (GAY), Obama (OBA), and marijuana (MAR). " + ] + }, + "annotation_id": "31c82ccf7692b2fe2aa345c66a99dbe4ff88eaac", + "worker_id": "c1018a31c3272ce74964a3280069f62f314a1a58" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "abortion (ABO), gay rights (GAY), Obama (OBA), and marijuana (MAR)" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "The CreateDebate dataset was collected from an English online debate forum discussing four topics: abortion (ABO), gay rights (GAY), Obama (OBA), and marijuana (MAR). The posts are annotated as for (F) and against (A). Replies to posts in this dataset are also labeled with stance and hence use the same data format as posts. The labeling results are shown in the right-hand side of Table TABREF12 . We observe that the dataset is more balanced than the FBFans dataset. In addition, there are 977 unique users in the dataset. To compare with Hasan and Ng's work, we conducted five-fold cross-validation and present the annotation results as the average number of all folds BIBREF9 , BIBREF5 ." + ], + "highlighted_evidence": [ + "The CreateDebate dataset was collected from an English online debate forum discussing four topics: abortion (ABO), gay rights (GAY), Obama (OBA), and marijuana (MAR). The posts are annotated as for (F) and against (A). Replies to posts in this dataset are also labeled with stance and hence use the same data format as posts. " + ] + }, + "annotation_id": "e713cf9d9a988c51b2bdad4b3173cad93d75b72c", + "worker_id": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4" + } + ] + }, + { + "question": "What is the size of the Chinese data?", + "question_id": "6a31db1aca57a818f36bba9002561724655372a7", + "nlp_background": "", + "topic_background": "", + "paper_read": "", + "search_query": "social media", + "question_writer": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "32,595 posts" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "To test whether the assumption of this paper – posts attract users who hold the same stance to like them – is reliable, we examine the likes from authors of different stances. Posts in FBFans dataset are used for this analysis. We calculate the like statistics of each distinct author from these 32,595 posts. As the numbers of authors in the Sup, Neu and Uns stances are largely imbalanced, these numbers are normalized by the number of users of each stance. Table TABREF13 shows the results. Posts with stances (i.e., not neutral) attract users of the same stance. Neutral posts also attract both supportive and neutral users, like what we observe in supportive posts, but just the neutral posts can attract even more neutral likers. These results do suggest that users prefer posts of the same stance, or at least posts of no obvious stance which might cause annoyance when reading, and hence support the user modeling in our approach." + ], + "highlighted_evidence": [ + "Posts in FBFans dataset are used for this analysis. We calculate the like statistics of each distinct author from these 32,595 posts." + ] + }, + "annotation_id": "421b4a1cb5e655bf876e28a979df32f623188040", + "worker_id": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "32,595" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "The FBFans dataset contains data from anti-nuclear-power Chinese Facebook fan groups from September 2013 to August 2014, including posts and their author and liker IDs. There are a total of 2,496 authors, 505,137 likers, 33,686 commenters, and 505,412 unique users. Two annotators were asked to take into account only the post content to label the stance of the posts in the whole dataset as supportive, neutral, or unsupportive (hereafter denoted as Sup, Neu, and Uns). Sup/Uns posts were those in support of or against anti-reconstruction; Neu posts were those evincing a neutral standpoint on the topic, or were irrelevant. Raw agreement between annotators is 0.91, indicating high agreement. Specifically, Cohen’s Kappa for Neu and not Neu labeling is 0.58 (moderate), and for Sup or Uns labeling is 0.84 (almost perfect). Posts with inconsistent labels were filtered out, and the development and testing sets were randomly selected from what was left. Posts in the development and testing sets involved at least one user who appeared in the training set. The number of posts for each stance is shown on the left-hand side of Table TABREF12 . About twenty percent of the posts were labeled with a stance, and the number of supportive (Sup) posts was much larger than that of the unsupportive (Uns) ones: this is thus highly skewed data, which complicates stance classification. On average, 161.1 users were involved in one post. The maximum was 23,297 and the minimum was one (the author). For comments, on average there were 3 comments per post. The maximum was 1,092 and the minimum was zero.", + "To test whether the assumption of this paper – posts attract users who hold the same stance to like them – is reliable, we examine the likes from authors of different stances. Posts in FBFans dataset are used for this analysis. We calculate the like statistics of each distinct author from these 32,595 posts. As the numbers of authors in the Sup, Neu and Uns stances are largely imbalanced, these numbers are normalized by the number of users of each stance. Table TABREF13 shows the results. Posts with stances (i.e., not neutral) attract users of the same stance. Neutral posts also attract both supportive and neutral users, like what we observe in supportive posts, but just the neutral posts can attract even more neutral likers. These results do suggest that users prefer posts of the same stance, or at least posts of no obvious stance which might cause annoyance when reading, and hence support the user modeling in our approach." + ], + "highlighted_evidence": [ + "The FBFans dataset contains data from anti-nuclear-power Chinese Facebook fan groups from September 2013 to August 2014, including posts and their author and liker IDs. ", + "Posts in FBFans dataset are used for this analysis. We calculate the like statistics of each distinct author from these 32,595 posts." + ] + }, + "annotation_id": "60930a2202aee84276d2f6d27ea3bb4a760b232b", + "worker_id": "c1018a31c3272ce74964a3280069f62f314a1a58" + } + ] + }, + { + "question": "Did they collected the two datasets?", + "question_id": "e330e162ec29722f5ec9f83853d129c9e0693d65", + "nlp_background": "", + "topic_background": "", + "paper_read": "", + "search_query": "social media", + "question_writer": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": false, + "free_form_answer": "", + "evidence": [], + "highlighted_evidence": [] + }, + "annotation_id": "03dce6f4b0fbb16248d2ea38c7b852f41d761554", + "worker_id": "c1018a31c3272ce74964a3280069f62f314a1a58" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": false, + "free_form_answer": "", + "evidence": [ + "We tested the proposed UTCNN on two different datasets: FBFans and CreateDebate. FBFans is a privately-owned, single-topic, Chinese, unbalanced, social media dataset, and CreateDebate is a public, multiple-topic, English, balanced, forum dataset. Results using these two datasets show the applicability and superiority for different topics, languages, data distributions, and platforms.", + "The CreateDebate dataset was collected from an English online debate forum discussing four topics: abortion (ABO), gay rights (GAY), Obama (OBA), and marijuana (MAR). The posts are annotated as for (F) and against (A). Replies to posts in this dataset are also labeled with stance and hence use the same data format as posts. The labeling results are shown in the right-hand side of Table TABREF12 . We observe that the dataset is more balanced than the FBFans dataset. In addition, there are 977 unique users in the dataset. To compare with Hasan and Ng's work, we conducted five-fold cross-validation and present the annotation results as the average number of all folds BIBREF9 , BIBREF5 ." + ], + "highlighted_evidence": [ + "We tested the proposed UTCNN on two different datasets: FBFans and CreateDebate. FBFans is a privately-owned, single-topic, Chinese, unbalanced, social media dataset, and CreateDebate is a public, multiple-topic, English, balanced, forum dataset. ", + "The CreateDebate dataset was collected from an English online debate forum discussing four topics: abortion (ABO), gay rights (GAY), Obama (OBA), and marijuana (MAR). The posts are annotated as for (F) and against (A). Replies to posts in this dataset are also labeled with stance and hence use the same data format as posts. The labeling results are shown in the right-hand side of Table TABREF12 . We observe that the dataset is more balanced than the FBFans dataset. In addition, there are 977 unique users in the dataset. To compare with Hasan and Ng's work, we conducted five-fold cross-validation and present the annotation results as the average number of all folds BIBREF9 , BIBREF5 ." + ] + }, + "annotation_id": "66bdb4452665b85b38877093b5a9388674bf7fb8", + "worker_id": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4" + } + ] + }, + { + "question": "What are the baselines?", + "question_id": "d3093062aebff475b4deab90815004051e802aa6", + "nlp_background": "", + "topic_background": "", + "paper_read": "", + "search_query": "social media", + "question_writer": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "SVM with unigram, bigram, and trigram features", + "SVM with average word embedding", + "SVM with average transformed word embeddings", + "CNN", + "ecurrent Convolutional Neural Networks", + "SVM and deep learning models with comment information" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "We pit our model against the following baselines: 1) SVM with unigram, bigram, and trigram features, which is a standard yet rather strong classifier for text features; 2) SVM with average word embedding, where a document is represented as a continuous representation by averaging the embeddings of the composite words; 3) SVM with average transformed word embeddings (the INLINEFORM0 in equation EQREF6 ), where a document is represented as a continuous representation by averaging the transformed embeddings of the composite words; 4) two mature deep learning models on text classification, CNN BIBREF3 and Recurrent Convolutional Neural Networks (RCNN) BIBREF0 , where the hyperparameters are based on their work; 5) the above SVM and deep learning models with comment information; 6) UTCNN without user information, representing a pure-text CNN model where we use the same user matrix and user embeddings INLINEFORM1 and INLINEFORM2 for each user; 7) UTCNN without the LDA model, representing how UTCNN works with a single-topic dataset; 8) UTCNN without comments, in which the model predicts the stance label given only user and topic information. All these models were trained on the training set, and parameters as well as the SVM kernel selections (linear or RBF) were fine-tuned on the development set. Also, we adopt oversampling on SVMs, CNN and RCNN because the FBFans dataset is highly imbalanced." + ], + "highlighted_evidence": [ + "We pit our model against the following baselines: 1) SVM with unigram, bigram, and trigram features, which is a standard yet rather strong classifier for text features; 2) SVM with average word embedding, where a document is represented as a continuous representation by averaging the embeddings of the composite words; 3) SVM with average transformed word embeddings (the INLINEFORM0 in equation EQREF6 ), where a document is represented as a continuous representation by averaging the transformed embeddings of the composite words; 4) two mature deep learning models on text classification, CNN BIBREF3 and Recurrent Convolutional Neural Networks (RCNN) BIBREF0 , where the hyperparameters are based on their work; 5) the above SVM and deep learning models with comment information; " + ] + }, + "annotation_id": "64b8c2449eb3d1583d86783d28f7a4764075495f", + "worker_id": "c1018a31c3272ce74964a3280069f62f314a1a58" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "SVM with unigram, bigram, trigram features, with average word embedding, with average transformed word embeddings, CNN and RCNN, SVM, CNN, RCNN with comment information", + "evidence": [ + "We pit our model against the following baselines: 1) SVM with unigram, bigram, and trigram features, which is a standard yet rather strong classifier for text features; 2) SVM with average word embedding, where a document is represented as a continuous representation by averaging the embeddings of the composite words; 3) SVM with average transformed word embeddings (the INLINEFORM0 in equation EQREF6 ), where a document is represented as a continuous representation by averaging the transformed embeddings of the composite words; 4) two mature deep learning models on text classification, CNN BIBREF3 and Recurrent Convolutional Neural Networks (RCNN) BIBREF0 , where the hyperparameters are based on their work; 5) the above SVM and deep learning models with comment information; 6) UTCNN without user information, representing a pure-text CNN model where we use the same user matrix and user embeddings INLINEFORM1 and INLINEFORM2 for each user; 7) UTCNN without the LDA model, representing how UTCNN works with a single-topic dataset; 8) UTCNN without comments, in which the model predicts the stance label given only user and topic information. All these models were trained on the training set, and parameters as well as the SVM kernel selections (linear or RBF) were fine-tuned on the development set. Also, we adopt oversampling on SVMs, CNN and RCNN because the FBFans dataset is highly imbalanced." + ], + "highlighted_evidence": [ + "We pit our model against the following baselines: 1) SVM with unigram, bigram, and trigram features, which is a standard yet rather strong classifier for text features; 2) SVM with average word embedding, where a document is represented as a continuous representation by averaging the embeddings of the composite words; 3) SVM with average transformed word embeddings (the INLINEFORM0 in equation EQREF6 ), where a document is represented as a continuous representation by averaging the transformed embeddings of the composite words; 4) two mature deep learning models on text classification, CNN BIBREF3 and Recurrent Convolutional Neural Networks (RCNN) BIBREF0 , where the hyperparameters are based on their work; 5) the above SVM and deep learning models with comment information; 6) UTCNN without user information, representing a pure-text CNN model where we use the same user matrix and user embeddings INLINEFORM1 and INLINEFORM2 for each user; 7) UTCNN without the LDA model, representing how UTCNN works with a single-topic dataset; 8) UTCNN without comments, in which the model predicts the stance label given only user and topic information. All these models were trained on the training set, and parameters as well as the SVM kernel selections (linear or RBF) were fine-tuned on the development set. Also, we adopt oversampling on SVMs, CNN and RCNN because the FBFans dataset is highly imbalanced." + ] + }, + "annotation_id": "d10bc18f3f2e713b53868da880c65e43da2fdd4a", + "worker_id": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4" + } + ] + } + ], + "figures_and_tables": [ + { + "file": "3-Figure1-1.png", + "caption": "Figure 1: Document composition in a convolutional neural network with three convolutional filters and user- and topic-dependent semantic transformations. Respectively, xw is the word embedding of word w, x′w is the word embedding of word w after transformation, Uk and Tj are user and topic matrix embeddings for user k and topic j." + }, + { + "file": "4-Figure2-1.png", + "caption": "Figure 2: The UTCNN model. Assuming one post author, l likers and p topics, xdw is the word embedding of word w in the document; xcw is the word embedding of word w in the comments; Uk and uk are the moderator matrix and vector embedding for moderator k; Tj and tj are the topic matrix and vector embedding for topic j; Ri and ri are the commenter matrix and vector embedding for commenter i. For simplicity we do not explicitly plot the topic vector embedding part for comments, but it does include a maximum pooling layer as with documents." + }, + { + "file": "5-Table1-1.png", + "caption": "Table 1: Annotation results of FBFans and CreateDebate dataset." + }, + { + "file": "5-Table2-1.png", + "caption": "Table 2: Distribution of like behavior." + }, + { + "file": "7-Table3-1.png", + "caption": "Table 3: Performance of post stance classification on the FBFans dataset. *UTCNN (full) results are statistically significant (p-value < 0.005) with respect to all other methods except for UTCNN shared user embedding." + }, + { + "file": "9-Table4-1.png", + "caption": "Table 4: Accuracies of post stance classification on CreateDebate dataset. *UTCNN results were statistically significant (p-value < 0.001) with respect to other UTCNN settings." + } + ] + }, + "1901.01010": { + "title": "A Joint Model for Multimodal Document Quality Assessment", + "abstract": "The quality of a document is affected by various factors, including grammaticality, readability, stylistics, and expertise depth, making the task of document quality assessment a complex one. In this paper, we explore this task in the context of assessing the quality of Wikipedia articles and academic papers. Observing that the visual rendering of a document can capture implicit quality indicators that are not present in the document text --- such as images, font choices, and visual layout --- we propose a joint model that combines the text content with a visual rendering of the document for document quality assessment. Experimental results over two datasets reveal that textual and visual features are complementary, achieving state-of-the-art results.", + "full_text": [ + { + "section_name": "Introduction", + "paragraphs": [ + "The task of document quality assessment is to automatically assess a document according to some predefined inventory of quality labels. This can take many forms, including essay scoring (quality = language quality, coherence, and relevance to a topic), job application filtering (quality = suitability for role + visual/presentational quality of the application), or answer selection in community question answering (quality = actionability + relevance of the answer to the question). In the case of this paper, we focus on document quality assessment in two contexts: Wikipedia document quality classification, and whether a paper submitted to a conference was accepted or not.", + "Automatic quality assessment has obvious benefits in terms of time savings and tractability in contexts where the volume of documents is large. In the case of dynamic documents (possibly with multiple authors), such as in the case of Wikipedia, it is particularly pertinent, as any edit potentially has implications for the quality label of that document (and around 10 English Wikipedia documents are edited per second). Furthermore, when the quality assessment task is decentralized (as in the case of Wikipedia and academic paper assessment), quality criteria are often applied inconsistently by different people, where an automatic document quality assessment system could potentially reduce inconsistencies and enable immediate author feedback.", + "Current studies on document quality assessment mainly focus on textual features. For example, BIBREF0 examine features such as the article length and the number of headings to predict the quality class of a Wikipedia article. In contrast to these studies, in this paper, we propose to combine text features with visual features, based on a visual rendering of the document. Figure 1 illustrates our intuition, relative to Wikipedia articles. Without being able to read the text, we can tell that the article in Figure 1 has higher quality than Figure 1 , as it has a detailed infobox, extensive references, and a variety of images. Based on this intuition, we aim to answer the following question: can we achieve better accuracy on document quality assessment by complementing textual features with visual features?", + "Our visual model is based on fine-tuning an Inception V3 model BIBREF1 over visual renderings of documents, while our textual model is based on a hierarchical biLSTM. We further combine the two into a joint model. We perform experiments on two datasets: a Wikipedia dataset novel to this paper, and an arXiv dataset provided by BIBREF2 split into three sub-parts based on subject category. Experimental results on the visual renderings of documents show that implicit quality indicators, such as images and visual layout, can be captured by an image classifier, at a level comparable to a text classifier. When we combine the two models, we achieve state-of-the-art results over 3/4 of our datasets.", + "This paper makes the following contributions:", + "All code and data associated with this research will be released on publication." + ] + }, + { + "section_name": "Related Work", + "paragraphs": [ + "A variety of approaches have been proposed for document quality assessment across different domains: Wikipedia article quality assessment, academic paper rating, content quality assessment in community question answering (cQA), and essay scoring. Among these approaches, some use hand-crafted features while others use neural networks to learn features from documents. For each domain, we first briefly describe feature-based approaches and then review neural network-based approaches. Wikipedia article quality assessment: Quality assessment of Wikipedia articles is a task that assigns a quality class label to a given Wikipedia article, mirroring the quality assessment process that the Wikipedia community carries out manually. Many approaches have been proposed that use features from the article itself, meta-data features (e.g., the editors, and Wikipedia article revision history), or a combination of the two. Article-internal features capture information such as whether an article is properly organized, with supporting evidence, and with appropriate terminology. For example, BIBREF3 use writing styles represented by binarized character trigram features to identify featured articles. BIBREF4 and BIBREF0 explore the number of headings, images, and references in the article. BIBREF5 use nine readability scores, such as the percentage of difficult words in the document, to measure the quality of the article. Meta-data features, which are indirect indicators of article quality, are usually extracted from revision history, and the interaction between editors and articles. For example, one heuristic that has been proposed is that higher-quality articles have more edits BIBREF6 , BIBREF7 . BIBREF8 use the percentage of registered editors and the total number of editors of an article. Article–editor dependencies have also been explored. For example, BIBREF9 use the authority of editors to measure the quality of Wikipedia articles, where the authority of editors is determined by the articles they edit. Deep learning approaches to predicting Wikipedia article quality have also been proposed. For example, BIBREF10 use a version of doc2vec BIBREF11 to represent articles, and feed the document embeddings into a four hidden layer neural network. BIBREF12 first obtain sentence representations by averaging words within a sentence, and then apply a biLSTM BIBREF13 to learn a document-level representation, which is combined with hand-crafted features as side information. BIBREF14 exploit two stacked biLSTMs to learn document representations.", + "Academic paper rating: Academic paper rating is a relatively new task in NLP/AI, with the basic formulation being to automatically predict whether to accept or reject a paper. BIBREF2 explore hand-crafted features, such as the length of the title, whether specific words (such as outperform, state-of-the-art, and novel) appear in the abstract, and an embedded representation of the abstract as input to different downstream learners, such as logistic regression, decision tree, and random forest. BIBREF15 exploit a modularized hierarchical convolutional neural network (CNN), where each paper section is treated as a module. For each paper section, they train an attention-based CNN, and an attentive pooling layer is applied to the concatenated representation of each section, which is then fed into a softmax layer.", + "Content quality assessment in cQA: Automatic quality assessment in cQA is the task of determining whether an answer is of high quality, selected as the best answer, or ranked higher than other answers. To measure answer content quality in cQA, researchers have exploited various features from different sources, such as the answer content itself, the answerer's profile, interactions among users, and usage of the content. The most common feature used is the answer length BIBREF16 , BIBREF17 , with other features including: syntactic and semantic features, such as readability scores. BIBREF18 ; similarity between the question and the answer at lexical, syntactic, and semantic levels BIBREF18 , BIBREF19 , BIBREF20 ; or user data (e.g., a user's status points or the number of answers written by the user). There have also been approaches using neural networks. For example, BIBREF21 combine CNN-learned representations with hand-crafted features to predict answer quality. BIBREF22 use a 2-dimensional CNN to learn the semantic relevance of an answer to the question, and apply an LSTM to the answer sequence to model thread context. BIBREF23 and BIBREF24 model the problem similarly to machine translation quality estimation, treating answers as competing translation hypotheses and the question as the reference translation, and apply neural machine translation to the problem. Essay scoring: Automated essay scoring is the task of assigning a score to an essay, usually in the context of assessing the language ability of a language learner. The quality of an essay is affected by the following four primary dimensions: topic relevance, organization and coherence, word usage and sentence complexity, and grammar and mechanics. To measure whether an essay is relevant to its “prompt” (the description of the essay topic), lexical and semantic overlap is commonly used BIBREF25 , BIBREF26 . BIBREF27 explore word features, such as the number of verb formation errors, average word frequency, and average word length, to measure word usage and lexical complexity. BIBREF28 use sentence structure features to measure sentence variety. The effects of grammatical and mechanic errors on the quality of an essay are measured via word and part-of-speech $n$ -gram features and “mechanics” features BIBREF29 (e.g., spelling, capitalization, and punctuation), respectively. BIBREF30 , BIBREF31 , and BIBREF32 use an LSTM to obtain an essay representation, which is used as the basis for classification. Similarly, BIBREF33 utilize a CNN to obtain sentence representation and an LSTM to obtain essay representation, with an attention layer at both the sentence and essay levels." + ] + }, + { + "section_name": "The Proposed Joint Model", + "paragraphs": [ + "We treat document quality assessment as a classification problem, i.e., given a document, we predict its quality class (e.g., whether an academic paper should be accepted or rejected). The proposed model is a joint model that integrates visual features learned through Inception V3 with textual features learned through a biLSTM. In this section, we present the details of the visual and textual embeddings, and finally describe how we combine the two. We return to discuss hyper-parameter settings and the experimental configuration in the Experiments section." + ] + }, + { + "section_name": "Visual Embedding Learning", + "paragraphs": [ + "A wide range of models have been proposed to tackle the image classification task, such as VGG BIBREF34 , ResNet BIBREF35 , Inception V3 BIBREF1 , and Xception BIBREF36 . However, to the best of our knowledge, there is no existing work that has proposed to use visual renderings of documents to assess document quality. In this paper, we use Inception V3 pretrained on ImageNet (“Inception” hereafter) to obtain visual embeddings of documents, noting that any image classifier could be applied to our task. The input to Inception is a visual rendering (screenshot) of a document, and the output is a visual embedding, which we will later integrate with our textual embedding.", + "Based on the observation that it is difficult to decide what types of convolution to apply to each layer (such as 3 $\\times $ 3 or 5 $\\times $ 5), the basic Inception model applies multiple convolution filters in parallel and concatenates the resulting features, which are fed into the next layer. This has the benefit of capturing both local features through smaller convolutions and abstracted features through larger convolutions. Inception is a hybrid of multiple Inception models of different architectures. To reduce computational cost, Inception also modifies the basic model by applying a 1 $\\times $ 1 convolution to the input and factorizing larger convolutions into smaller ones." + ] + }, + { + "section_name": "Textual Embedding Learning", + "paragraphs": [ + "We adopt a bi-directional LSTM model to generate textual embeddings for document quality assessment, following the method of BIBREF12 (“biLSTM” hereafter). The input to biLSTM is a textual document, and the output is a textual embedding, which will later integrate with the visual embedding.", + "For biLSTM, each word is represented as a word embedding BIBREF37 , and an average-pooling layer is applied to the word embeddings to obtain the sentence embedding, which is fed into a bi-directional LSTM to generate the document embedding from the sentence embeddings. Then a max-pooling layer is applied to select the most salient features from the component sentences." + ] + }, + { + "section_name": "The Joint Model", + "paragraphs": [ + "The proposed joint model (“Joint” hereafter) combines the visual and textual embeddings (output of Inception and biLSTM) via a simple feed-forward layer and softmax over the document label set, as shown in Figure 2 . We optimize our model based on cross-entropy loss." + ] + }, + { + "section_name": "Experiments", + "paragraphs": [ + "In this section, we first describe the two datasets used in our experiments: (1) Wikipedia, and (2) arXiv. Then, we report the experimental details and results." + ] + }, + { + "section_name": "Datasets", + "paragraphs": [ + "The Wikipedia dataset consists of articles from English Wikipedia, with quality class labels assigned by the Wikipedia community. Wikipedia articles are labelled with one of six quality classes, in descending order of quality: Featured Article (“FA”), Good Article (“GA”), B-class Article (“B”), C-class Article (“C”), Start Article (“Start”), and Stub Article (“Stub”). A description of the criteria associated with the different classes can be found in the Wikipedia grading scheme page. The quality class of a Wikipedia article is assigned by Wikipedia reviewers or any registered user, who can discuss through the article's talk page to reach consensus. We constructed the dataset by first crawling all articles from each quality class repository, e.g., we get FA articles by crawling pages from the FA repository: https://en.wikipedia.org/wiki/Category:Featured_articles. This resulted in around 5K FA, 28K GA, 212K B, 533K C, 2.6M Start, and 3.2M Stub articles.", + "We randomly sampled 5,000 articles from each quality class and removed all redirect pages, resulting in a dataset of 29,794 articles. As the wikitext contained in each document contains markup relating to the document category such as {Featured Article} or {geo-stub}, which reveals the label, we remove such information. We additionally randomly partitioned this dataset into training, development, and test splits based on a ratio of 8:1:1. Details of the dataset are summarized in Table 1 .", + "We generate a visual representation of each document via a 1,000 $\\times $ 2,000-pixel screenshot of the article via a PhantomJS script over the rendered version of the article, ensuring that the screenshot and wikitext versions of the article are the same version. Any direct indicators of document quality (such as the FA indicator, which is a bronze star icon in the top right corner of the webpage) are removed from the screenshot.", + "The arXiv dataset BIBREF2 consists of three subsets of academic articles under the arXiv repository of Computer Science (cs), from the three subject areas of: Artificial Intelligence (cs.ai), Computation and Language (cs.cl), and Machine Learning (cs.lg). In line with the original dataset formulation BIBREF2 , a paper is considered to have been accepted (i.e. is positively labeled) if it matches a paper in the DBLP database or is otherwise accepted by any of the following conferences: ACL, EMNLP, NAACL, EACL, TACL, NIPS, ICML, ICLR, or AAAI. Failing this, it is considered to be rejected (noting that some of the papers may not have been submitted to one of these conferences). The median numbers of pages for papers in cs.ai, cs.cl, and cs.lg are 11, 10, and 12, respectively. To make sure each page in the PDF file has the same size in the screenshot, we crop the PDF file of a paper to the first 12; we pad the PDF file with blank pages if a PDF file has less than 12 pages, using the PyPDF2 Python package. We then use ImageMagick to convert the 12-page PDF file to a single 1,000 $\\times $ 2,000 pixel screenshot. Table 2 details this dataset, where the “Accepted” column denotes the percentage of positive instances (accepted papers) in each subset." + ] + }, + { + "section_name": "Experimental Setting", + "paragraphs": [ + "As discussed above, our model has two main components — biLSTM and Inception— which generate textual and visual representations, respectively. For the biLSTM component, the documents are preprocessed as described in BIBREF12 , where an article is divided into sentences and tokenized using NLTK BIBREF38 . Words appearing more than 20 times are retained when building the vocabulary. All other words are replaced by the special UNK token. We use the pre-trained GloVe BIBREF39 50-dimensional word embeddings to represent words. For words not in GloVe, word embeddings are randomly initialized based on sampling from a uniform distribution $U(-1, 1)$ . All word embeddings are updated in the training process. We set the LSTM hidden layer size to 256. The concatenation of the forward and backward LSTMs thus gives us 512 dimensions for the document embedding. A dropout layer is applied at the sentence and document level, respectively, with a probability of 0.5.", + "For Inception, we adopt data augmentation techniques in the training with a “nearest” filling mode, a zoom range of 0.1, a width shift range of 0.1, and a height shift range of 0.1. As the original screenshots have the size of 1,000 $\\times 2$ ,000 pixels, they are resized to 500 $\\times $ 500 to feed into Inception, where the input shape is (500, 500, 3). A dropout layer is applied with a probability of 0.5. Then, a GlobalAveragePooling2D layer is applied, which produces a 2,048 dimensional representation.", + "For the Joint model, we get a representation of 2,560 dimensions by concatenating the 512 dimensional representation from the biLSTM with the 2,048 dimensional representation from Inception. The dropout layer is applied to the two components with a probability of 0.5. For biLSTM, we use a mini-batch size of 128 and a learning rate of 0.001. For both Inception and joint model, we use a mini-batch size of 16 and a learning rate of 0.0001. All hyper-parameters were set empirically over the development data, and the models were optimized using the Adam optimizer BIBREF40 .", + "In the training phase, the weights in Inception are initialized by parameters pretrained on ImageNet, and the weights in biLSTM are randomly initialized (except for the word embeddings). We train each model for 50 epochs. However, to prevent overfitting, we adopt early stopping, where we stop training the model if the performance on the development set does not improve for 20 epochs. For evaluation, we use (micro-)accuracy, following previous studies BIBREF5 , BIBREF2 ." + ] + }, + { + "section_name": "Baseline Approaches", + "paragraphs": [ + "We compare our models against the following five baselines:", + "Majority: the model labels all test samples with the majority class of the training data.", + "Benchmark: a benchmark method from the literature. In the case of Wikipedia, this is BIBREF5 , who use structural features and readability scores as features to build a random forest classifier; for arXiv, this is BIBREF2 , who use hand-crafted features, such as the number of references and TF-IDF weighted bag-of-words in abstract, to build a classifier based on the best of logistic regression, multi-layer perception, and AdaBoost.", + "Doc2Vec: doc2vec BIBREF11 to learn document embeddings with a dimension of 500, and a 4-layer feed-forward classification model on top of this, with 2000, 1000, 500, and 200 dimensions, respectively.", + "biLSTM: first derive a sentence representation by averaging across words in a sentence, then feed the sentence representation into a biLSTM and a maxpooling layer over output sequence to learn a document level representation with a dimension of 512, which is used to predict document quality.", + "Inception $_{\\text{fixed}}$ : the frozen Inception model, where only parameters in the last layer are fine-tuned during training.", + "The hyper-parameters of Benchmark, Doc2Vec, and biLSTM are based on the corresponding papers except that: (1) we fine-tune the feed forward layer of Doc2Vec on the development set and train the model 300 epochs on Wikipedia and 50 epochs on arXiv; (2) we do not use hand-crafted features for biLSTM as we want the baselines to be comparable to our models, and the main focus of this paper is not to explore the effects of hand-crafted features (e.g., see BIBREF12 )." + ] + }, + { + "section_name": "Experimental Results", + "paragraphs": [ + "Table 3 shows the performance of the different models over our two datasets, in the form of the average accuracy on the test set (along with the standard deviation) over 10 runs, with different random initializations.", + "On Wikipedia, we observe that the performance of biLSTM, Inception, and Joint is much better than that of all four baselines. Inception achieves 2.9% higher accuracy than biLSTM. The performance of Joint achieves an accuracy of 59.4%, which is 5.3% higher than using textual features alone (biLSTM) and 2.4% higher than using visual features alone (Inception). Based on a one-tailed Wilcoxon signed-rank test, the performance of Joint is statistically significant ( $p<0.05$ ). This shows that the textual and visual features complement each other, achieving state-of-the-art results in combination.", + "For arXiv, baseline methods Majority, Benchmark, and Inception $_{\\text{fixed}}$ outperform biLSTM over cs.ai, in large part because of the class imbalance in this dataset (90% of papers are rejected). Surprisingly, Inception $_{\\text{fixed}}$ is better than Majority and Benchmark over the arXiv cs.lg subset, which verifies the usefulness of visual features, even when only the last layer is fine-tuned. Table 3 also shows that Inception and biLSTM achieve similar performance on arXiv, showing that textual and visual representations are equally discriminative: Inception and biLSTM are indistinguishable over cs.cl; biLSTM achieves 1.8% higher accuracy over cs.lg, while Inception achieves 1.3% higher accuracy over cs.ai. Once again, the Joint model achieves the highest accuracy on cs.ai and cs.cl by combining textual and visual representations (at a level of statistical significance for cs.ai). This, again, confirms that textual and visual features complement each other, and together they achieve state-of-the-art results. On arXiv cs.lg, Joint achieves a 0.6% higher accuracy than Inception by combining visual features and textual features, but biLSTM achieves the highest accuracy. One characteristic of cs.lg documents is that they tend to contain more equations than the other two arXiv datasets, and preliminary analysis suggests that the biLSTM is picking up on a correlation between the volume/style of mathematical presentation and the quality of the document." + ] + }, + { + "section_name": "Analysis", + "paragraphs": [ + "In this section, we first analyze the performance of Inception and Joint. We also analyze the performance of different models on different quality classes. The high-level representations learned by different models are also visualized and discussed. As the Wikipedia test set is larger and more balanced than that of arXiv, our analysis will focus on Wikipedia." + ] + }, + { + "section_name": "Inception", + "paragraphs": [ + "To better understand the performance of Inception, we generated the gradient-based class activation map BIBREF41 , by maximizing the outputs of each class in the penultimate layer, as shown in Figure 3 . From Figure 3 and Figure 3 , we can see that Inception identifies the two most important regions (one at the top corresponding to the table of contents, and the other at the bottom, capturing both document length and references) that contribute to the FA class prediction, and a region in the upper half of the image that contributes to the GA class prediction (capturing the length of the article body). From Figure 3 and Figure 3 , we can see that the most important regions in terms of B and C class prediction capture images (down the left and right of the page, in the case of B and C), and document length/references. From Figure 3 and Figure 3 , we can see that Inception finds that images in the top right corner are the strongest predictor of Start class prediction, and (the lack of) images/the link bar down the left side of the document are the most important for Stub class prediction." + ] + }, + { + "section_name": "Joint", + "paragraphs": [ + "Table 4 shows the confusion matrix of Joint on Wikipedia. We can see that more than 50% of documents for each quality class are correctly classified, except for the C class where more documents are misclassified into B. Analysis shows that when misclassified, documents are usually misclassified into adjacent quality classes, which can be explained by the Wikipedia grading scheme, where the criteria for adjacent quality classes are more similar.", + "We also provide a breakdown of precision (“ $\\mathcal {P}$ ”), recall (“ $\\mathcal {R}$ ”), and F1 score (“ $\\mathcal {F}_{\\beta =1}$ ”) for biLSTM, Inception, and Joint across the quality classes in Table 5 . We can see that Joint achieves the highest accuracy in 11 out of 18 cases. It is also worth noting that all models achieve higher scores for FA, GA, and Stub articles than B, C and Start articles. This can be explained in part by the fact that FA and GA articles must pass an official review based on structured criteria, and in part by the fact that Stub articles are usually very short, which is discriminative for Inception, and Joint. All models perform worst on the B and C quality classes. It is difficult to differentiate B articles from C articles even for Wikipedia contributors. As evidence of this, when we crawled a new dataset including talk pages with quality class votes from Wikipedia contributors, we found that among articles with three or more quality labels, over 20% percent of B and C articles have inconsistent votes from Wikipedia contributors, whereas for FA and GA articles the number is only 0.7%.", + "We further visualize the learned document representations of biLSTM, Inception, and Joint in the form of a t-SNE plot BIBREF42 in Figure 4 . The degree of separation between Start and Stub achieved by Inception is much greater than for biLSTM, with the separation between Start and Stub achieved by Joint being the clearest among the three models. Inception and Joint are better than biLSTM at separating Start and C. Joint achieves slightly better performance than Inception in separating GA and FA. We can also see that it is difficult for all models to separate B and C, which is consistent with the findings of Tables 4 and 5 ." + ] + }, + { + "section_name": "Conclusions", + "paragraphs": [ + "We proposed to use visual renderings of documents to capture implicit document quality indicators, such as font choices, images, and visual layout, which are not captured in textual content. We applied neural network models to capture visual features given visual renderings of documents. Experimental results show that we achieve a 2.9% higher accuracy than state-of-the-art approaches based on textual features over Wikipedia, and performance competitive with or surpassing state-of-the-art approaches over arXiv. We further proposed a joint model, combining textual and visual representations, to predict the quality of a document. Experimental results show that our joint model outperforms the visual-only model in all cases, and the text-only model on Wikipedia and two subsets of arXiv. These results underline the feasibility of assessing document quality via visual features, and the complementarity of visual and textual document representations for quality assessment." + ] + } + ], + "qas": [ + { + "question": "Which fonts are the best indicators of high quality?", + "question_id": "e438445cf823893c841b2bc26cdce32ccc3f5cbe", + "nlp_background": "infinity", + "topic_background": "familiar", + "paper_read": "no", + "search_query": "wikipedia", + "answers": [ + { + "answer": { + "unanswerable": true, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "", + "evidence": [], + "highlighted_evidence": [] + }, + "annotation_id": "5c89df6284cd22b539a9f6613273c5fab1e2a8f6", + "worker_id": "1ba1b5b562aef9cd264cace5b7bdd46a7c065c0a" + } + ], + "question_writer": "50d8b4a941c26b89482c94ab324b5a274f9ced66" + }, + { + "question": "What kind of model do they use?", + "question_id": "12f7fac818f0006cf33269c9eafd41bbb8979a48", + "nlp_background": "infinity", + "topic_background": "familiar", + "paper_read": "no", + "search_query": "wikipedia", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "visual model is based on fine-tuning an Inception V3 model BIBREF1 over visual renderings of documents, while our textual model is based on a hierarchical biLSTM. We further combine the two into a joint model. ", + "neural network models" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "Our visual model is based on fine-tuning an Inception V3 model BIBREF1 over visual renderings of documents, while our textual model is based on a hierarchical biLSTM. We further combine the two into a joint model. We perform experiments on two datasets: a Wikipedia dataset novel to this paper, and an arXiv dataset provided by BIBREF2 split into three sub-parts based on subject category. Experimental results on the visual renderings of documents show that implicit quality indicators, such as images and visual layout, can be captured by an image classifier, at a level comparable to a text classifier. When we combine the two models, we achieve state-of-the-art results over 3/4 of our datasets.", + "We proposed to use visual renderings of documents to capture implicit document quality indicators, such as font choices, images, and visual layout, which are not captured in textual content. We applied neural network models to capture visual features given visual renderings of documents. Experimental results show that we achieve a 2.9% higher accuracy than state-of-the-art approaches based on textual features over Wikipedia, and performance competitive with or surpassing state-of-the-art approaches over arXiv. We further proposed a joint model, combining textual and visual representations, to predict the quality of a document. Experimental results show that our joint model outperforms the visual-only model in all cases, and the text-only model on Wikipedia and two subsets of arXiv. These results underline the feasibility of assessing document quality via visual features, and the complementarity of visual and textual document representations for quality assessment." + ], + "highlighted_evidence": [ + "Our visual model is based on fine-tuning an Inception V3 model BIBREF1 over visual renderings of documents, while our textual model is based on a hierarchical biLSTM. We further combine the two into a joint model. ", + "We applied neural network models to capture visual features given visual renderings of documents. Experimental results show that we achieve a 2.9% higher accuracy than state-of-the-art approaches based on textual features over Wikipedia, and performance competitive with or surpassing state-of-the-art approaches over arXiv. We further proposed a joint model, combining textual and visual representations, to predict the quality of a document. " + ] + }, + "annotation_id": "316e121b1e4fd2ca8a99cb9fc9ae3d8b55250ca5", + "worker_id": "1ba1b5b562aef9cd264cace5b7bdd46a7c065c0a" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "Inception V3", + "biLSTM" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "We treat document quality assessment as a classification problem, i.e., given a document, we predict its quality class (e.g., whether an academic paper should be accepted or rejected). The proposed model is a joint model that integrates visual features learned through Inception V3 with textual features learned through a biLSTM. In this section, we present the details of the visual and textual embeddings, and finally describe how we combine the two. We return to discuss hyper-parameter settings and the experimental configuration in the Experiments section." + ], + "highlighted_evidence": [ + "The proposed model is a joint model that integrates visual features learned through Inception V3 with textual features learned through a biLSTM." + ] + }, + "annotation_id": "7c61fe9072dbbffabed837c8c2ade9cce21a70d6", + "worker_id": "18f4d5a2eb93a969d55361267e74aa0c4f6f82fe" + } + ], + "question_writer": "50d8b4a941c26b89482c94ab324b5a274f9ced66" + }, + { + "question": "Did they release their data set of academic papers?", + "question_id": "d5a8fd8bb48dd1f75927e874bdea582b4732a0cd", + "nlp_background": "infinity", + "topic_background": "familiar", + "paper_read": "no", + "search_query": "wikipedia", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": false, + "free_form_answer": "", + "evidence": [], + "highlighted_evidence": [] + }, + "annotation_id": "0974b080edadfde80be0ee44ba02f449181fcc24", + "worker_id": "18f4d5a2eb93a969d55361267e74aa0c4f6f82fe" + }, + { + "answer": { + "unanswerable": true, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "", + "evidence": [], + "highlighted_evidence": [] + }, + "annotation_id": "81411c4526e552b8003efe274f0a5e70f37f6284", + "worker_id": "1ba1b5b562aef9cd264cace5b7bdd46a7c065c0a" + } + ], + "question_writer": "50d8b4a941c26b89482c94ab324b5a274f9ced66" + }, + { + "question": "Do the methods that work best on academic papers also work best on Wikipedia?", + "question_id": "1097768b89f8bd28d6ef6443c94feb04c1a1318e", + "nlp_background": "infinity", + "topic_background": "familiar", + "paper_read": "no", + "search_query": "wikipedia", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": true, + "free_form_answer": "", + "evidence": [ + "We proposed to use visual renderings of documents to capture implicit document quality indicators, such as font choices, images, and visual layout, which are not captured in textual content. We applied neural network models to capture visual features given visual renderings of documents. Experimental results show that we achieve a 2.9% higher accuracy than state-of-the-art approaches based on textual features over Wikipedia, and performance competitive with or surpassing state-of-the-art approaches over arXiv. We further proposed a joint model, combining textual and visual representations, to predict the quality of a document. Experimental results show that our joint model outperforms the visual-only model in all cases, and the text-only model on Wikipedia and two subsets of arXiv. These results underline the feasibility of assessing document quality via visual features, and the complementarity of visual and textual document representations for quality assessment." + ], + "highlighted_evidence": [ + "Experimental results show that our joint model outperforms the visual-only model in all cases, and the text-only model on Wikipedia and two subsets of arXiv." + ] + }, + "annotation_id": "3ac21164fc20e40db8abff77a1facf6ee0f31b5f", + "worker_id": "18f4d5a2eb93a969d55361267e74aa0c4f6f82fe" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": false, + "free_form_answer": "", + "evidence": [ + "We proposed to use visual renderings of documents to capture implicit document quality indicators, such as font choices, images, and visual layout, which are not captured in textual content. We applied neural network models to capture visual features given visual renderings of documents. Experimental results show that we achieve a 2.9% higher accuracy than state-of-the-art approaches based on textual features over Wikipedia, and performance competitive with or surpassing state-of-the-art approaches over arXiv. We further proposed a joint model, combining textual and visual representations, to predict the quality of a document. Experimental results show that our joint model outperforms the visual-only model in all cases, and the text-only model on Wikipedia and two subsets of arXiv. These results underline the feasibility of assessing document quality via visual features, and the complementarity of visual and textual document representations for quality assessment.", + "FLOAT SELECTED: Table 1: Experimental results. The best result for each dataset is indicated in bold, and marked with “†” if it is significantly higher than the second best result (based on a one-tailed Wilcoxon signed-rank test; p < 0.05). The results of Benchmark on Peer Review are from the original paper, where the standard deviation values were not reported." + ], + "highlighted_evidence": [ + "We applied neural network models to capture visual features given visual renderings of documents. Experimental results show that we achieve a 2.9% higher accuracy than state-of-the-art approaches based on textual features over Wikipedia, and performance competitive with or surpassing state-of-the-art approaches over arXiv. We further proposed a joint model, combining textual and visual representations, to predict the quality of a document. Experimental results show that our joint model outperforms the visual-only model in all cases, and the text-only model on Wikipedia and two subsets of arXiv. ", + "FLOAT SELECTED: Table 1: Experimental results. The best result for each dataset is indicated in bold, and marked with “†” if it is significantly higher than the second best result (based on a one-tailed Wilcoxon signed-rank test; p < 0.05). The results of Benchmark on Peer Review are from the original paper, where the standard deviation values were not reported." + ] + }, + "annotation_id": "b950b110bcde785db7c6dd0cc0fea4d8027ef76d", + "worker_id": "1ba1b5b562aef9cd264cace5b7bdd46a7c065c0a" + } + ], + "question_writer": "50d8b4a941c26b89482c94ab324b5a274f9ced66" + }, + { + "question": "What is their system's absolute accuracy?", + "question_id": "fc1679c714eab822431bbe96f0e9cf4079cd8b8d", + "nlp_background": "infinity", + "topic_background": "familiar", + "paper_read": "no", + "search_query": "wikipedia", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "59.4% on wikipedia dataset, 93.4% on peer-reviewed archive AI papers, 77.1% on peer-reviewed archive Computation and Language papers, and 79.9% on peer-reviewed archive Machine Learning papers", + "evidence": [ + "FLOAT SELECTED: Table 1: Experimental results. The best result for each dataset is indicated in bold, and marked with “†” if it is significantly higher than the second best result (based on a one-tailed Wilcoxon signed-rank test; p < 0.05). The results of Benchmark on Peer Review are from the original paper, where the standard deviation values were not reported." + ], + "highlighted_evidence": [ + "FLOAT SELECTED: Table 1: Experimental results. The best result for each dataset is indicated in bold, and marked with “†” if it is significantly higher than the second best result (based on a one-tailed Wilcoxon signed-rank test; p < 0.05). The results of Benchmark on Peer Review are from the original paper, where the standard deviation values were not reported." + ] + }, + "annotation_id": "ed1a1df25ec1afd52388f24b9d8281481956d845", + "worker_id": "1ba1b5b562aef9cd264cace5b7bdd46a7c065c0a" + } + ], + "question_writer": "50d8b4a941c26b89482c94ab324b5a274f9ced66" + }, + { + "question": "Which is more useful, visual or textual features?", + "question_id": "23e2971c962bb6486bc0a66ff04242170dd22a1d", + "nlp_background": "infinity", + "topic_background": "familiar", + "paper_read": "no", + "search_query": "wikipedia", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "It depends on the dataset. Experimental results over two datasets reveal that textual and visual features are complementary. ", + "evidence": [ + "We proposed to use visual renderings of documents to capture implicit document quality indicators, such as font choices, images, and visual layout, which are not captured in textual content. We applied neural network models to capture visual features given visual renderings of documents. Experimental results show that we achieve a 2.9% higher accuracy than state-of-the-art approaches based on textual features over Wikipedia, and performance competitive with or surpassing state-of-the-art approaches over arXiv. We further proposed a joint model, combining textual and visual representations, to predict the quality of a document. Experimental results show that our joint model outperforms the visual-only model in all cases, and the text-only model on Wikipedia and two subsets of arXiv. These results underline the feasibility of assessing document quality via visual features, and the complementarity of visual and textual document representations for quality assessment." + ], + "highlighted_evidence": [ + "We proposed to use visual renderings of documents to capture implicit document quality indicators, such as font choices, images, and visual layout, which are not captured in textual content. We applied neural network models to capture visual features given visual renderings of documents. Experimental results show that we achieve a 2.9% higher accuracy than state-of-the-art approaches based on textual features over Wikipedia, and performance competitive with or surpassing state-of-the-art approaches over arXiv. We further proposed a joint model, combining textual and visual representations, to predict the quality of a document. Experimental results show that our joint model outperforms the visual-only model in all cases, and the text-only model on Wikipedia and two subsets of arXiv. These results underline the feasibility of assessing document quality via visual features, and the complementarity of visual and textual document representations for quality assessment." + ] + }, + "annotation_id": "11320bef73e775a60a772fc0e3f732ec8e5ade41", + "worker_id": "1ba1b5b562aef9cd264cace5b7bdd46a7c065c0a" + } + ], + "question_writer": "50d8b4a941c26b89482c94ab324b5a274f9ced66" + }, + { + "question": "Which languages do they use?", + "question_id": "c9bc6f53b941863e801280343afa14248521ce43", + "nlp_background": "infinity", + "topic_background": "familiar", + "paper_read": "no", + "search_query": "wikipedia", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "English" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "The Wikipedia dataset consists of articles from English Wikipedia, with quality class labels assigned by the Wikipedia community. Wikipedia articles are labelled with one of six quality classes, in descending order of quality: Featured Article (“FA”), Good Article (“GA”), B-class Article (“B”), C-class Article (“C”), Start Article (“Start”), and Stub Article (“Stub”). A description of the criteria associated with the different classes can be found in the Wikipedia grading scheme page. The quality class of a Wikipedia article is assigned by Wikipedia reviewers or any registered user, who can discuss through the article's talk page to reach consensus. We constructed the dataset by first crawling all articles from each quality class repository, e.g., we get FA articles by crawling pages from the FA repository: https://en.wikipedia.org/wiki/Category:Featured_articles. This resulted in around 5K FA, 28K GA, 212K B, 533K C, 2.6M Start, and 3.2M Stub articles." + ], + "highlighted_evidence": [ + "The Wikipedia dataset consists of articles from English Wikipedia, with quality class labels assigned by the Wikipedia community." + ] + }, + "annotation_id": "3422ddce21d18df60177fea06b9eba6bdea5dbb8", + "worker_id": "18f4d5a2eb93a969d55361267e74aa0c4f6f82fe" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "English", + "evidence": [ + "The Wikipedia dataset consists of articles from English Wikipedia, with quality class labels assigned by the Wikipedia community. Wikipedia articles are labelled with one of six quality classes, in descending order of quality: Featured Article (“FA”), Good Article (“GA”), B-class Article (“B”), C-class Article (“C”), Start Article (“Start”), and Stub Article (“Stub”). A description of the criteria associated with the different classes can be found in the Wikipedia grading scheme page. The quality class of a Wikipedia article is assigned by Wikipedia reviewers or any registered user, who can discuss through the article's talk page to reach consensus. We constructed the dataset by first crawling all articles from each quality class repository, e.g., we get FA articles by crawling pages from the FA repository: https://en.wikipedia.org/wiki/Category:Featured_articles. This resulted in around 5K FA, 28K GA, 212K B, 533K C, 2.6M Start, and 3.2M Stub articles.", + "The arXiv dataset BIBREF2 consists of three subsets of academic articles under the arXiv repository of Computer Science (cs), from the three subject areas of: Artificial Intelligence (cs.ai), Computation and Language (cs.cl), and Machine Learning (cs.lg). In line with the original dataset formulation BIBREF2 , a paper is considered to have been accepted (i.e. is positively labeled) if it matches a paper in the DBLP database or is otherwise accepted by any of the following conferences: ACL, EMNLP, NAACL, EACL, TACL, NIPS, ICML, ICLR, or AAAI. Failing this, it is considered to be rejected (noting that some of the papers may not have been submitted to one of these conferences). The median numbers of pages for papers in cs.ai, cs.cl, and cs.lg are 11, 10, and 12, respectively. To make sure each page in the PDF file has the same size in the screenshot, we crop the PDF file of a paper to the first 12; we pad the PDF file with blank pages if a PDF file has less than 12 pages, using the PyPDF2 Python package. We then use ImageMagick to convert the 12-page PDF file to a single 1,000 $\\times $ 2,000 pixel screenshot. Table 2 details this dataset, where the “Accepted” column denotes the percentage of positive instances (accepted papers) in each subset." + ], + "highlighted_evidence": [ + "The Wikipedia dataset consists of articles from English Wikipedia, with quality class labels assigned by the Wikipedia community. ", + "The arXiv dataset BIBREF2 consists of three subsets of academic articles under the arXiv repository of Computer Science (cs), from the three subject areas of: Artificial Intelligence (cs.ai), Computation and Language (cs.cl), and Machine Learning (cs.lg)." + ] + }, + "annotation_id": "4a69f21ca18ea894839b8f6e09455f5a87c07a39", + "worker_id": "1ba1b5b562aef9cd264cace5b7bdd46a7c065c0a" + } + ], + "question_writer": "50d8b4a941c26b89482c94ab324b5a274f9ced66" + }, + { + "question": "How large is their data set?", + "question_id": "07b70b2b799b9efa630e8737df8b1dd1284f032c", + "nlp_background": "infinity", + "topic_background": "familiar", + "paper_read": "no", + "search_query": "wikipedia", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "a sample of 29,794 wikipedia articles and 2,794 arXiv papers ", + "evidence": [ + "We randomly sampled 5,000 articles from each quality class and removed all redirect pages, resulting in a dataset of 29,794 articles. As the wikitext contained in each document contains markup relating to the document category such as {Featured Article} or {geo-stub}, which reveals the label, we remove such information. We additionally randomly partitioned this dataset into training, development, and test splits based on a ratio of 8:1:1. Details of the dataset are summarized in Table 1 .", + "The Wikipedia dataset consists of articles from English Wikipedia, with quality class labels assigned by the Wikipedia community. Wikipedia articles are labelled with one of six quality classes, in descending order of quality: Featured Article (“FA”), Good Article (“GA”), B-class Article (“B”), C-class Article (“C”), Start Article (“Start”), and Stub Article (“Stub”). A description of the criteria associated with the different classes can be found in the Wikipedia grading scheme page. The quality class of a Wikipedia article is assigned by Wikipedia reviewers or any registered user, who can discuss through the article's talk page to reach consensus. We constructed the dataset by first crawling all articles from each quality class repository, e.g., we get FA articles by crawling pages from the FA repository: https://en.wikipedia.org/wiki/Category:Featured_articles. This resulted in around 5K FA, 28K GA, 212K B, 533K C, 2.6M Start, and 3.2M Stub articles." + ], + "highlighted_evidence": [ + " 29,794", + "The Wikipedia dataset consists of articles from English Wikipedia, with quality class labels assigned by the Wikipedia community. Wikipedia articles are labelled with one of six quality classes, in descending order of quality: Featured Article (“FA”), Good Article (“GA”), B-class Article (“B”), C-class Article (“C”), Start Article (“Start”), and Stub Article (“Stub”).", + "We randomly sampled 5,000 articles from each quality class and removed all redirect pages, resulting in a dataset of 29,794 articles." + ] + }, + "annotation_id": "2cf6a4ccaa19d91c5dfe27d0ee203c3da4538346", + "worker_id": "1ba1b5b562aef9cd264cace5b7bdd46a7c065c0a" + } + ], + "question_writer": "50d8b4a941c26b89482c94ab324b5a274f9ced66" + }, + { + "question": "Where do they get their ground truth quality judgments?", + "question_id": "71a0c4f19be4ce1b1bae58a6e8f2a586e125d074", + "nlp_background": "infinity", + "topic_background": "familiar", + "paper_read": "no", + "search_query": "wikipedia", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "Wikipedia articles are labelled with one of six quality classes, in descending order of quality: Featured Article (“FA”), Good Article (“GA”), B-class Article (“B”), C-class Article (“C”), Start Article (“Start”), and Stub Article (“Stub”).", + "The quality class of a Wikipedia article is assigned by Wikipedia reviewers or any registered user, who can discuss through the article's talk page to reach consensus.", + "The arXiv dataset BIBREF2 consists of three subsets of academic articles under the arXiv repository of Computer Science (cs), from the three subject areas of: Artificial Intelligence (cs.ai), Computation and Language (cs.cl), and Machine Learning (cs.lg). In line with the original dataset formulation BIBREF2 , a paper is considered to have been accepted (i.e. is positively labeled) if it matches a paper in the DBLP database or is otherwise accepted by any of the following conferences: ACL, EMNLP, NAACL, EACL, TACL, NIPS, ICML, ICLR, or AAAI. Failing this, it is considered to be rejected (noting that some of the papers may not have been submitted to one of these conferences). " + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "The Wikipedia dataset consists of articles from English Wikipedia, with quality class labels assigned by the Wikipedia community. Wikipedia articles are labelled with one of six quality classes, in descending order of quality: Featured Article (“FA”), Good Article (“GA”), B-class Article (“B”), C-class Article (“C”), Start Article (“Start”), and Stub Article (“Stub”). A description of the criteria associated with the different classes can be found in the Wikipedia grading scheme page. The quality class of a Wikipedia article is assigned by Wikipedia reviewers or any registered user, who can discuss through the article's talk page to reach consensus. We constructed the dataset by first crawling all articles from each quality class repository, e.g., we get FA articles by crawling pages from the FA repository: https://en.wikipedia.org/wiki/Category:Featured_articles. This resulted in around 5K FA, 28K GA, 212K B, 533K C, 2.6M Start, and 3.2M Stub articles.", + "The arXiv dataset BIBREF2 consists of three subsets of academic articles under the arXiv repository of Computer Science (cs), from the three subject areas of: Artificial Intelligence (cs.ai), Computation and Language (cs.cl), and Machine Learning (cs.lg). In line with the original dataset formulation BIBREF2 , a paper is considered to have been accepted (i.e. is positively labeled) if it matches a paper in the DBLP database or is otherwise accepted by any of the following conferences: ACL, EMNLP, NAACL, EACL, TACL, NIPS, ICML, ICLR, or AAAI. Failing this, it is considered to be rejected (noting that some of the papers may not have been submitted to one of these conferences). The median numbers of pages for papers in cs.ai, cs.cl, and cs.lg are 11, 10, and 12, respectively. To make sure each page in the PDF file has the same size in the screenshot, we crop the PDF file of a paper to the first 12; we pad the PDF file with blank pages if a PDF file has less than 12 pages, using the PyPDF2 Python package. We then use ImageMagick to convert the 12-page PDF file to a single 1,000 $\\times $ 2,000 pixel screenshot. Table 2 details this dataset, where the “Accepted” column denotes the percentage of positive instances (accepted papers) in each subset." + ], + "highlighted_evidence": [ + "The quality class of a Wikipedia article is assigned by Wikipedia reviewers or any registered user, who can discuss through the article's talk page to reach consensus.", + "The arXiv dataset BIBREF2 consists of three subsets of academic articles under the arXiv repository of Computer Science (cs), from the three subject areas of: Artificial Intelligence (cs.ai), Computation and Language (cs.cl), and Machine Learning (cs.lg). In line with the original dataset formulation BIBREF2 , a paper is considered to have been accepted (i.e. is positively labeled) if it matches a paper in the DBLP database or is otherwise accepted by any of the following conferences: ACL, EMNLP, NAACL, EACL, TACL, NIPS, ICML, ICLR, or AAAI. Failing this, it is considered to be rejected (noting that some of the papers may not have been submitted to one of these conferences)." + ] + }, + "annotation_id": "e7d6a6f5b941e1092fd56036d45fd141999db228", + "worker_id": "1ba1b5b562aef9cd264cace5b7bdd46a7c065c0a" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "quality class labels assigned by the Wikipedia community", + "a paper is considered to have been accepted (i.e. is positively labeled) if it matches a paper in the DBLP database or is otherwise accepted by any of the following conferences: ACL, EMNLP, NAACL, EACL, TACL, NIPS, ICML, ICLR, or AAAI" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "The Wikipedia dataset consists of articles from English Wikipedia, with quality class labels assigned by the Wikipedia community. Wikipedia articles are labelled with one of six quality classes, in descending order of quality: Featured Article (“FA”), Good Article (“GA”), B-class Article (“B”), C-class Article (“C”), Start Article (“Start”), and Stub Article (“Stub”). A description of the criteria associated with the different classes can be found in the Wikipedia grading scheme page. The quality class of a Wikipedia article is assigned by Wikipedia reviewers or any registered user, who can discuss through the article's talk page to reach consensus. We constructed the dataset by first crawling all articles from each quality class repository, e.g., we get FA articles by crawling pages from the FA repository: https://en.wikipedia.org/wiki/Category:Featured_articles. This resulted in around 5K FA, 28K GA, 212K B, 533K C, 2.6M Start, and 3.2M Stub articles.", + "The arXiv dataset BIBREF2 consists of three subsets of academic articles under the arXiv repository of Computer Science (cs), from the three subject areas of: Artificial Intelligence (cs.ai), Computation and Language (cs.cl), and Machine Learning (cs.lg). In line with the original dataset formulation BIBREF2 , a paper is considered to have been accepted (i.e. is positively labeled) if it matches a paper in the DBLP database or is otherwise accepted by any of the following conferences: ACL, EMNLP, NAACL, EACL, TACL, NIPS, ICML, ICLR, or AAAI. Failing this, it is considered to be rejected (noting that some of the papers may not have been submitted to one of these conferences). The median numbers of pages for papers in cs.ai, cs.cl, and cs.lg are 11, 10, and 12, respectively. To make sure each page in the PDF file has the same size in the screenshot, we crop the PDF file of a paper to the first 12; we pad the PDF file with blank pages if a PDF file has less than 12 pages, using the PyPDF2 Python package. We then use ImageMagick to convert the 12-page PDF file to a single 1,000 $\\times $ 2,000 pixel screenshot. Table 2 details this dataset, where the “Accepted” column denotes the percentage of positive instances (accepted papers) in each subset." + ], + "highlighted_evidence": [ + "The Wikipedia dataset consists of articles from English Wikipedia, with quality class labels assigned by the Wikipedia community. Wikipedia articles are labelled with one of six quality classes, in descending order of quality: Featured Article (“FA”), Good Article (“GA”), B-class Article (“B”), C-class Article (“C”), Start Article (“Start”), and Stub Article (“Stub”).", + "In line with the original dataset formulation BIBREF2 , a paper is considered to have been accepted (i.e. is positively labeled) if it matches a paper in the DBLP database or is otherwise accepted by any of the following conferences: ACL, EMNLP, NAACL, EACL, TACL, NIPS, ICML, ICLR, or AAAI. Failing this, it is considered to be rejected (noting that some of the papers may not have been submitted to one of these conferences)." + ] + }, + "annotation_id": "f07ad9db9b91d3edf6db2365df8902f86d4a1629", + "worker_id": "18f4d5a2eb93a969d55361267e74aa0c4f6f82fe" + } + ], + "question_writer": "50d8b4a941c26b89482c94ab324b5a274f9ced66" + } + ], + "figures_and_tables": [ + { + "file": "1-Figure1-1.png", + "caption": "Figure 1: Visual renderings of two exampleWikipedia documents with different quality labels (not intended to be readable)." + }, + { + "file": "3-Figure2-1.png", + "caption": "Figure 2: Heatmap overlapped onto screenshots of FA and Stub. Best viewed in color." + }, + { + "file": "4-Table1-1.png", + "caption": "Table 1: Experimental results. The best result for each dataset is indicated in bold, and marked with “†” if it is significantly higher than the second best result (based on a one-tailed Wilcoxon signed-rank test; p < 0.05). The results of Benchmark on Peer Review are from the original paper, where the standard deviation values were not reported." + }, + { + "file": "4-Table2-1.png", + "caption": "Table 2: Confusion matrix of the Joint model onWikipedia. Rows are the actual quality classes and columns are the predicted quality classes. The gray cells are correct predictions." + } + ] + }, + "1902.09666": { + "title": "Predicting the Type and Target of Offensive Posts in Social Media", + "abstract": "As offensive content has become pervasive in social media, there has been much research in identifying potentially offensive messages. However, previous work on this topic did not consider the problem as a whole, but rather focused on detecting very specific types of offensive content, e.g., hate speech, cyberbulling, or cyber-aggression. In contrast, here we target several different kinds of offensive content. In particular, we model the task hierarchically, identifying the type and the target of offensive messages in social media. For this purpose, we complied the Offensive Language Identification Dataset (OLID), a new dataset with tweets annotated for offensive content using a fine-grained three-layer annotation scheme, which we make publicly available. We discuss the main similarities and differences between OLID and pre-existing datasets for hate speech identification, aggression detection, and similar tasks. We further experiment with and we compare the performance of different machine learning models on OLID.", + "full_text": [ + { + "section_name": "Introduction", + "paragraphs": [ + "Offensive content has become pervasive in social media and a reason of concern for government organizations, online communities, and social media platforms. One of the most common strategies to tackle the problem is to train systems capable of recognizing offensive content, which then can be deleted or set aside for human moderation. In the last few years, there have been several studies published on the application of computational methods to deal with this problem. Most prior work focuses on a different aspect of offensive language such as abusive language BIBREF0 , BIBREF1 , (cyber-)aggression BIBREF2 , (cyber-)bullying BIBREF3 , BIBREF4 , toxic comments INLINEFORM0 , hate speech BIBREF5 , BIBREF6 , BIBREF7 , BIBREF8 , BIBREF9 , BIBREF10 , and offensive language BIBREF11 . Prior work has focused on these aspects of offensive language in Twitter BIBREF3 , BIBREF7 , BIBREF8 , BIBREF11 , Wikipedia comments, and Facebook posts BIBREF2 .", + "Recently, Waseem et. al. ( BIBREF12 ) acknowledged the similarities among prior work and discussed the need for a typology that differentiates between whether the (abusive) language is directed towards a specific individual or entity or towards a generalized group and whether the abusive content is explicit or implicit. Wiegand et al. ( BIBREF11 ) followed this trend as well on German tweets. In their evaluation, they have a task to detect offensive vs not offensive tweets and a second task for distinguishing between the offensive tweets as profanity, insult, or abuse. However, no prior work has explored the target of the offensive language, which is important in many scenarios, e.g., when studying hate speech with respect to a specific target.", + "Therefore, we expand on these ideas by proposing a a hierarchical three-level annotation model that encompasses:", + "", + "", + "Using this annotation model, we create a new large publicly available dataset of English tweets. The key contributions of this paper are as follows:" + ] + }, + { + "section_name": "Related Work", + "paragraphs": [ + "Different abusive and offense language identification sub-tasks have been explored in the past few years including aggression identification, bullying detection, hate speech, toxic comments, and offensive language.", + "", + "Aggression identification: The TRAC shared task on Aggression Identification BIBREF2 provided participants with a dataset containing 15,000 annotated Facebook posts and comments in English and Hindi for training and validation. For testing, two different sets, one from Facebook and one from Twitter were provided. Systems were trained to discriminate between three classes: non-aggressive, covertly aggressive, and overtly aggressive.", + "", + "Bullying detection: Several studies have been published on bullying detection. One of them is the one by xu2012learning which apply sentiment analysis to detect bullying in tweets. xu2012learning use topic models to to identify relevant topics in bullying. Another related study is the one by dadvar2013improving which use user-related features such as the frequency of profanity in previous messages to improve bullying detection.", + "", + "Hate speech identification: It is perhaps the most widespread abusive language detection sub-task. There have been several studies published on this sub-task such as kwok2013locate and djuric2015hate who build a binary classifier to distinguish between `clean' comments and comments containing hate speech and profanity. More recently, Davidson et al. davidson2017automated presented the hate speech detection dataset containing over 24,000 English tweets labeled as non offensive, hate speech, and profanity.", + "", + "Offensive language: The GermEval BIBREF11 shared task focused on Offensive language identification in German tweets. A dataset of over 8,500 annotated tweets was provided for a course-grained binary classification task in which systems were trained to discriminate between offensive and non-offensive tweets and a second task where the organizers broke down the offensive class into three classes: profanity, insult, and abuse.", + "", + "Toxic comments: The Toxic Comment Classification Challenge was an open competition at Kaggle which provided participants with comments from Wikipedia labeled in six classes: toxic, severe toxic, obscene, threat, insult, identity hate.", + "", + "While each of these sub-tasks tackle a particular type of abuse or offense, they share similar properties and the hierarchical annotation model proposed in this paper aims to capture this. Considering that, for example, an insult targeted at an individual is commonly known as cyberbulling and that insults targeted at a group are known as hate speech, we pose that OLID's hierarchical annotation model makes it a useful resource for various offensive language identification sub-tasks." + ] + }, + { + "section_name": "Hierarchically Modelling Offensive Content", + "paragraphs": [ + "In the OLID dataset, we use a hierarchical annotation model split into three levels to distinguish between whether language is offensive or not (A), and type (B) and target (C) of the offensive language. Each level is described in more detail in the following subsections and examples are shown in Table TABREF10 ." + ] + }, + { + "section_name": "Level A: Offensive language Detection", + "paragraphs": [ + "Level A discriminates between offensive (OFF) and non-offensive (NOT) tweets.", + "", + "Not Offensive (NOT): Posts that do not contain offense or profanity;", + "Offensive (OFF): We label a post as offensive if it contains any form of non-acceptable language (profanity) or a targeted offense, which can be veiled or direct. This category includes insults, threats, and posts containing profane language or swear words." + ] + }, + { + "section_name": "Level B: Categorization of Offensive Language", + "paragraphs": [ + "Level B categorizes the type of offense and two labels are used: targeted (TIN) and untargeted (INT) insults and threats.", + "Targeted Insult (TIN): Posts which contain an insult/threat to an individual, group, or others (see next layer);", + "Untargeted (UNT): Posts containing non-targeted profanity and swearing. Posts with general profanity are not targeted, but they contain non-acceptable language." + ] + }, + { + "section_name": "Level C: Offensive Language Target Identification", + "paragraphs": [ + "Level C categorizes the targets of insults and threats as individual (IND), group (GRP), and other (OTH).", + "", + "Individual (IND): Posts targeting an individual. It can be a a famous person, a named individual or an unnamed participant in the conversation. Insults and threats targeted at individuals are often defined as cyberbulling.", + "Group (GRP): The target of these offensive posts is a group of people considered as a unity due to the same ethnicity, gender or sexual orientation, political affiliation, religious belief, or other common characteristic. Many of the insults and threats targeted at a group correspond to what is commonly understood as hate speech.", + "Other (OTH): The target of these offensive posts does not belong to any of the previous two categories (e.g. an organization, a situation, an event, or an issue)." + ] + }, + { + "section_name": "Data Collection", + "paragraphs": [ + "The data included in OLID has been collected from Twitter. We retrieved the data using the Twitter API by searching for keywords and constructions that are often included in offensive messages, such as `she is' or `to:BreitBartNews'. We carried out a first round of trial annotation of 300 instances with six experts. The goal of the trial annotation was to 1) evaluate the proposed tagset; 2) evaluate the data retrieval method; and 3) create a gold standard with instances that could be used as test questions in the training and test setting annotation which was carried out using crowdsourcing. The breakdown of keywords and their offensive content in the trial data of 300 tweets is shown in Table TABREF14 . We included a left (@NewYorker) and far-right (@BreitBartNews) news accounts because there tends to be political offense in the comments. One of the best offensive keywords was tweets that were flagged as not being safe by the Twitter `safe' filter (the `-' indicates `not safe'). The vast majority of content on Twitter is not offensive so we tried different strategies to keep a reasonable number of tweets in the offensive class amounting to around 30% of the dataset including excluding some keywords that were not high in offensive content such as `they are` and `to:NewYorker`. Although `he is' is lower in offensive content we kept it as a keyword to avoid gender bias. In addition to the keywords in the trial set, we searched for more political keywords which tend to be higher in offensive content, and sampled our dataset such that 50% of the the tweets come from political keywords and 50% come from non-political keywords. In addition to the keywords `gun control', and `to:BreitbartNews', political keywords used to collect these tweets are `MAGA', `antifa', `conservative' and `liberal'. We computed Fliess' INLINEFORM0 on the trial set for the five annotators on 21 of the tweets. INLINEFORM1 is .83 for Layer A (OFF vs NOT) indicating high agreement. As to normalization and anonymization, no user metadata or Twitter IDs have been stored, and URLs and Twitter mentions have been substituted to placeholders. We follow prior work in related areas (burnap2015cyber,davidson2017automated) and annotate our data using crowdsourcing using the platform Figure Eight. We ensure data quality by: 1) we only received annotations from individuals who were experienced in the platform; and 2) we used test questions to discard annotations of individuals who did not reach a certain threshold. Each instance in the dataset was annotated by multiple annotators and inter-annotator agreement has been calculated. We first acquired two annotations for each instance. In case of 100% agreement, we considered these as acceptable annotations, and in case of disagreement, we requested more annotations until the agreement was above 66%. After the crowdsourcing annotation, we used expert adjudication to guarantee the quality of the annotation. The breakdown of the data into training and testing for the labels from each level is shown in Table TABREF15 ." + ] + }, + { + "section_name": "Experiments and Evaluation", + "paragraphs": [ + "We assess our dataset using traditional and deep learning methods. Our simplest model is a linear SVM trained on word unigrams. SVMs have produced state-of-the-art results for many text classification tasks BIBREF13 . We also train a bidirectional Long Short-Term-Memory (BiLSTM) model, which we adapted from the sentiment analysis system of sentimentSystem,rasooli2018cross and altered to predict offensive labels instead. It consists of (1) an input embedding layer, (2) a bidirectional LSTM layer, (3) an average pooling layer of input features. The concatenation of the LSTM's and average pool layer is passed through a dense layer and the output is passed through a softmax function. We set two input channels for the input embedding layers: pre-trained FastText embeddings BIBREF14 , as well as updatable embeddings learned by the model during training. Finally, we also apply a Convolutional Neural Network (CNN) model based on the architecture of BIBREF15 , using the same multi-channel inputs as the above BiLSTM.", + "Our models are trained on the training data, and evaluated by predicting the labels for the held-out test set. The distribution is described in Table TABREF15 . We evaluate and compare the models using the macro-averaged F1-score as the label distribution is highly imbalanced. Per-class Precision (P), Recall (R), and F1-score (F1), also with other averaged metrics are also reported. The models are compared against baselines of predicting all labels as the majority or minority classes." + ] + }, + { + "section_name": "Offensive Language Detection", + "paragraphs": [ + "The performance on discriminating between offensive (OFF) and non-offensive (NOT) posts is reported in Table TABREF18 . We can see that all systems perform significantly better than chance, with the neural models being substantially better than the SVM. The CNN outperforms the RNN model, achieving a macro-F1 score of 0.80." + ] + }, + { + "section_name": "Categorization of Offensive Language", + "paragraphs": [ + "In this experiment, the two systems were trained to discriminate between insults and threats (TIN) and untargeted (UNT) offenses, which generally refer to profanity. The results are shown in Table TABREF19 .", + "The CNN system achieved higher performance in this experiment compared to the BiLSTM, with a macro-F1 score of 0.69. All systems performed better at identifying target and threats (TIN) than untargeted offenses (UNT)." + ] + }, + { + "section_name": "Offensive Language Target Identification", + "paragraphs": [ + "The results of the offensive target identification experiment are reported in Table TABREF20 . Here the systems were trained to distinguish between three targets: a group (GRP), an individual (IND), or others (OTH). All three models achieved similar results far surpassing the random baselines, with a slight performance edge for the neural models.", + "The performance of all systems for the OTH class is 0. This poor performances can be explained by two main factors. First, unlike the two other classes, OTH is a heterogeneous collection of targets. It includes offensive tweets targeted at organizations, situations, events, etc. making it more challenging for systems to learn discriminative properties of this class. Second, this class contains fewer training instances than the other two. There are only 395 instances in OTH, and 1,075 in GRP, and 2,407 in IND." + ] + }, + { + "section_name": "Conclusion and Future Work", + "paragraphs": [ + "This paper presents OLID, a new dataset with annotation of type and target of offensive language. OLID is the official dataset of the shared task SemEval 2019 Task 6: Identifying and Categorizing Offensive Language in Social Media (OffensEval) BIBREF16 . In OffensEval, each annotation level in OLID is an independent sub-task. The dataset contains 14,100 tweets and is released freely to the research community. To the best of our knowledge, this is the first dataset to contain annotation of type and target of offenses in social media and it opens several new avenues for research in this area. We present baseline experiments using SVMs and neural networks to identify the offensive tweets, discriminate between insults, threats, and profanity, and finally to identify the target of the offensive messages. The results show that this is a challenging task. A CNN-based sentence classifier achieved the best results in all three sub-tasks.", + "In future work, we would like to make a cross-corpus comparison of OLID and datasets annotated for similar tasks such as aggression identification BIBREF2 and hate speech detection BIBREF8 . This comparison is, however, far from trivial as the annotation of OLID is different." + ] + }, + { + "section_name": "Acknowledgments", + "paragraphs": [ + "The research presented in this paper was partially supported by an ERAS fellowship awarded by the University of Wolverhampton." + ] + } + ], + "qas": [ + { + "question": "What models are used in the experiment?", + "question_id": "8c852fc29bda014d28c3ee5b5a7e449ab9152d35", + "nlp_background": "", + "topic_background": "", + "paper_read": "", + "search_query": "", + "question_writer": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "linear SVM", + "bidirectional Long Short-Term-Memory (BiLSTM)", + "Convolutional Neural Network (CNN)" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "We assess our dataset using traditional and deep learning methods. Our simplest model is a linear SVM trained on word unigrams. SVMs have produced state-of-the-art results for many text classification tasks BIBREF13 . We also train a bidirectional Long Short-Term-Memory (BiLSTM) model, which we adapted from the sentiment analysis system of sentimentSystem,rasooli2018cross and altered to predict offensive labels instead. It consists of (1) an input embedding layer, (2) a bidirectional LSTM layer, (3) an average pooling layer of input features. The concatenation of the LSTM's and average pool layer is passed through a dense layer and the output is passed through a softmax function. We set two input channels for the input embedding layers: pre-trained FastText embeddings BIBREF14 , as well as updatable embeddings learned by the model during training. Finally, we also apply a Convolutional Neural Network (CNN) model based on the architecture of BIBREF15 , using the same multi-channel inputs as the above BiLSTM." + ], + "highlighted_evidence": [ + "We assess our dataset using traditional and deep learning methods. Our simplest model is a linear SVM trained on word unigrams. SVMs have produced state-of-the-art results for many text classification tasks BIBREF13 . We also train a bidirectional Long Short-Term-Memory (BiLSTM) model, which we adapted from the sentiment analysis system of sentimentSystem,rasooli2018cross and altered to predict offensive labels instead. It consists of (1) an input embedding layer, (2) a bidirectional LSTM layer, (3) an average pooling layer of input features. The concatenation of the LSTM's and average pool layer is passed through a dense layer and the output is passed through a softmax function. We set two input channels for the input embedding layers: pre-trained FastText embeddings BIBREF14 , as well as updatable embeddings learned by the model during training. Finally, we also apply a Convolutional Neural Network (CNN) model based on the architecture of BIBREF15 , using the same multi-channel inputs as the above BiLSTM." + ] + }, + "annotation_id": "710b79c1764a5c374181d22c40405e48e46c9475", + "worker_id": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "linear SVM", + "bidirectional Long Short-Term-Memory (BiLSTM)", + "Convolutional Neural Network (CNN)" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "We assess our dataset using traditional and deep learning methods. Our simplest model is a linear SVM trained on word unigrams. SVMs have produced state-of-the-art results for many text classification tasks BIBREF13 . We also train a bidirectional Long Short-Term-Memory (BiLSTM) model, which we adapted from the sentiment analysis system of sentimentSystem,rasooli2018cross and altered to predict offensive labels instead. It consists of (1) an input embedding layer, (2) a bidirectional LSTM layer, (3) an average pooling layer of input features. The concatenation of the LSTM's and average pool layer is passed through a dense layer and the output is passed through a softmax function. We set two input channels for the input embedding layers: pre-trained FastText embeddings BIBREF14 , as well as updatable embeddings learned by the model during training. Finally, we also apply a Convolutional Neural Network (CNN) model based on the architecture of BIBREF15 , using the same multi-channel inputs as the above BiLSTM." + ], + "highlighted_evidence": [ + "Our simplest model is a linear SVM trained on word unigrams. SVMs have produced state-of-the-art results for many text classification tasks BIBREF13 . We also train a bidirectional Long Short-Term-Memory (BiLSTM) model, which we adapted from the sentiment analysis system of sentimentSystem,rasooli2018cross and altered to predict offensive labels instead.", + "Finally, we also apply a Convolutional Neural Network (CNN) model based on the architecture of BIBREF15 , using the same multi-channel inputs as the above BiLSTM." + ] + }, + "annotation_id": "7cb14177df857a2fc6924f9ee1db0da32bc85869", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "linear SVM trained on word unigrams", + " bidirectional Long Short-Term-Memory (BiLSTM)", + " Convolutional Neural Network (CNN) " + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "We assess our dataset using traditional and deep learning methods. Our simplest model is a linear SVM trained on word unigrams. SVMs have produced state-of-the-art results for many text classification tasks BIBREF13 . We also train a bidirectional Long Short-Term-Memory (BiLSTM) model, which we adapted from the sentiment analysis system of sentimentSystem,rasooli2018cross and altered to predict offensive labels instead. It consists of (1) an input embedding layer, (2) a bidirectional LSTM layer, (3) an average pooling layer of input features. The concatenation of the LSTM's and average pool layer is passed through a dense layer and the output is passed through a softmax function. We set two input channels for the input embedding layers: pre-trained FastText embeddings BIBREF14 , as well as updatable embeddings learned by the model during training. Finally, we also apply a Convolutional Neural Network (CNN) model based on the architecture of BIBREF15 , using the same multi-channel inputs as the above BiLSTM." + ], + "highlighted_evidence": [ + "We assess our dataset using traditional and deep learning methods. Our simplest model is a linear SVM trained on word unigrams. SVMs have produced state-of-the-art results for many text classification tasks BIBREF13 . We also train a bidirectional Long Short-Term-Memory (BiLSTM) model, which we adapted from the sentiment analysis system of sentimentSystem,rasooli2018cross and altered to predict offensive labels instead. It consists of (1) an input embedding layer, (2) a bidirectional LSTM layer, (3) an average pooling layer of input features. The concatenation of the LSTM's and average pool layer is passed through a dense layer and the output is passed through a softmax function. We set two input channels for the input embedding layers: pre-trained FastText embeddings BIBREF14 , as well as updatable embeddings learned by the model during training. Finally, we also apply a Convolutional Neural Network (CNN) model based on the architecture of BIBREF15 , using the same multi-channel inputs as the above BiLSTM." + ] + }, + "annotation_id": "de250af74bdaa41eddbbf03be791a39fd51f2de6", + "worker_id": "34c35a1877e453ecaebcf625df3ef788e1953cc4" + } + ] + }, + { + "question": "What are the differences between this dataset and pre-existing ones?", + "question_id": "682e26262abba473412f68cbeb5f69aa3b9968d7", + "nlp_background": "", + "topic_background": "", + "paper_read": "", + "search_query": "", + "question_writer": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "no prior work has explored the target of the offensive language" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "Recently, Waseem et. al. ( BIBREF12 ) acknowledged the similarities among prior work and discussed the need for a typology that differentiates between whether the (abusive) language is directed towards a specific individual or entity or towards a generalized group and whether the abusive content is explicit or implicit. Wiegand et al. ( BIBREF11 ) followed this trend as well on German tweets. In their evaluation, they have a task to detect offensive vs not offensive tweets and a second task for distinguishing between the offensive tweets as profanity, insult, or abuse. However, no prior work has explored the target of the offensive language, which is important in many scenarios, e.g., when studying hate speech with respect to a specific target." + ], + "highlighted_evidence": [ + "However, no prior work has explored the target of the offensive language, which is important in many scenarios, e.g., when studying hate speech with respect to a specific target." + ] + }, + "annotation_id": "985b8f9bb01b53a734f42b520af29870f4e077b0", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + } + ] + }, + { + "question": "In what language are the tweets?", + "question_id": "5daeb8d4d6f3b8543ec6309a7a35523e160437eb", + "nlp_background": "", + "topic_background": "", + "paper_read": "", + "search_query": "", + "question_writer": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "English" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "Using this annotation model, we create a new large publicly available dataset of English tweets. The key contributions of this paper are as follows:" + ], + "highlighted_evidence": [ + "Using this annotation model, we create a new large publicly available dataset of English tweets." + ] + }, + "annotation_id": "2b209806dc5b32658f12ec18c449b1cc667a85e7", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "English " + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "Using this annotation model, we create a new large publicly available dataset of English tweets. The key contributions of this paper are as follows:" + ], + "highlighted_evidence": [ + "Using this annotation model, we create a new large publicly available dataset of English tweets. " + ] + }, + "annotation_id": "61eaf62bda7a05c65a5f7469673929c38dab40c9", + "worker_id": "34c35a1877e453ecaebcf625df3ef788e1953cc4" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "English" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "Using this annotation model, we create a new large publicly available dataset of English tweets. The key contributions of this paper are as follows:" + ], + "highlighted_evidence": [ + "Using this annotation model, we create a new large publicly available dataset of English tweets. " + ] + }, + "annotation_id": "956b5562b42da93ca0508613501dccb4d3f9b5b6", + "worker_id": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4" + } + ] + }, + { + "question": "What is the size of the new dataset?", + "question_id": "74fb77a624ea9f1821f58935a52cca3086bb0981", + "nlp_background": "", + "topic_background": "", + "paper_read": "", + "search_query": "", + "question_writer": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4", + "answers": [ + { + "answer": { + "unanswerable": true, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "", + "evidence": [], + "highlighted_evidence": [] + }, + "annotation_id": "06bef9f965e0df0274090562f8345d3d9180a544", + "worker_id": "34c35a1877e453ecaebcf625df3ef788e1953cc4" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "14,100 tweets", + "evidence": [ + "FLOAT SELECTED: Table 3: Distribution of label combinations in OLID." + ], + "highlighted_evidence": [ + "FLOAT SELECTED: Table 3: Distribution of label combinations in OLID." + ] + }, + "annotation_id": "b250fd2019421911362cc78e7fe0696615ed0284", + "worker_id": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "Dataset contains total of 14100 annotations.", + "evidence": [ + "FLOAT SELECTED: Table 3: Distribution of label combinations in OLID.", + "The data included in OLID has been collected from Twitter. We retrieved the data using the Twitter API by searching for keywords and constructions that are often included in offensive messages, such as `she is' or `to:BreitBartNews'. We carried out a first round of trial annotation of 300 instances with six experts. The goal of the trial annotation was to 1) evaluate the proposed tagset; 2) evaluate the data retrieval method; and 3) create a gold standard with instances that could be used as test questions in the training and test setting annotation which was carried out using crowdsourcing. The breakdown of keywords and their offensive content in the trial data of 300 tweets is shown in Table TABREF14 . We included a left (@NewYorker) and far-right (@BreitBartNews) news accounts because there tends to be political offense in the comments. One of the best offensive keywords was tweets that were flagged as not being safe by the Twitter `safe' filter (the `-' indicates `not safe'). The vast majority of content on Twitter is not offensive so we tried different strategies to keep a reasonable number of tweets in the offensive class amounting to around 30% of the dataset including excluding some keywords that were not high in offensive content such as `they are` and `to:NewYorker`. Although `he is' is lower in offensive content we kept it as a keyword to avoid gender bias. In addition to the keywords in the trial set, we searched for more political keywords which tend to be higher in offensive content, and sampled our dataset such that 50% of the the tweets come from political keywords and 50% come from non-political keywords. In addition to the keywords `gun control', and `to:BreitbartNews', political keywords used to collect these tweets are `MAGA', `antifa', `conservative' and `liberal'. We computed Fliess' INLINEFORM0 on the trial set for the five annotators on 21 of the tweets. INLINEFORM1 is .83 for Layer A (OFF vs NOT) indicating high agreement. As to normalization and anonymization, no user metadata or Twitter IDs have been stored, and URLs and Twitter mentions have been substituted to placeholders. We follow prior work in related areas (burnap2015cyber,davidson2017automated) and annotate our data using crowdsourcing using the platform Figure Eight. We ensure data quality by: 1) we only received annotations from individuals who were experienced in the platform; and 2) we used test questions to discard annotations of individuals who did not reach a certain threshold. Each instance in the dataset was annotated by multiple annotators and inter-annotator agreement has been calculated. We first acquired two annotations for each instance. In case of 100% agreement, we considered these as acceptable annotations, and in case of disagreement, we requested more annotations until the agreement was above 66%. After the crowdsourcing annotation, we used expert adjudication to guarantee the quality of the annotation. The breakdown of the data into training and testing for the labels from each level is shown in Table TABREF15 ." + ], + "highlighted_evidence": [ + "FLOAT SELECTED: Table 3: Distribution of label combinations in OLID.", + "The breakdown of the data into training and testing for the labels from each level is shown in Table TABREF15 ." + ] + }, + "annotation_id": "fb2149e3fa1fc8db4bb7dd2c8d79ac81ca9ea1c4", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + } + ] + }, + { + "question": "What kinds of offensive content are explored?", + "question_id": "d015faf0f8dcf2e15c1690bbbe2bf1e7e0ce3751", + "nlp_background": "", + "topic_background": "", + "paper_read": "", + "search_query": "", + "question_writer": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "non-targeted profanity and swearing, targeted insults such as cyberbullying, offensive content related to ethnicity, gender or sexual orientation, political affiliation, religious belief, and anything belonging to hate speech", + "evidence": [ + "Level B categorizes the type of offense and two labels are used: targeted (TIN) and untargeted (INT) insults and threats.", + "Targeted Insult (TIN): Posts which contain an insult/threat to an individual, group, or others (see next layer);", + "Untargeted (UNT): Posts containing non-targeted profanity and swearing. Posts with general profanity are not targeted, but they contain non-acceptable language.", + "Level C categorizes the targets of insults and threats as individual (IND), group (GRP), and other (OTH).", + "Individual (IND): Posts targeting an individual. It can be a a famous person, a named individual or an unnamed participant in the conversation. Insults and threats targeted at individuals are often defined as cyberbulling.", + "Group (GRP): The target of these offensive posts is a group of people considered as a unity due to the same ethnicity, gender or sexual orientation, political affiliation, religious belief, or other common characteristic. Many of the insults and threats targeted at a group correspond to what is commonly understood as hate speech.", + "Other (OTH): The target of these offensive posts does not belong to any of the previous two categories (e.g. an organization, a situation, an event, or an issue)." + ], + "highlighted_evidence": [ + "Level B categorizes the type of offense and two labels are used: targeted (TIN) and untargeted (INT) insults and threats.\n\nTargeted Insult (TIN): Posts which contain an insult/threat to an individual, group, or others (see next layer);\n\nUntargeted (UNT): Posts containing non-targeted profanity and swearing. Posts with general profanity are not targeted, but they contain non-acceptable language.", + "Level C categorizes the targets of insults and threats as individual (IND), group (GRP), and other (OTH).\n\nIndividual (IND): Posts targeting an individual. It can be a a famous person, a named individual or an unnamed participant in the conversation. Insults and threats targeted at individuals are often defined as cyberbulling.\n\nGroup (GRP): The target of these offensive posts is a group of people considered as a unity due to the same ethnicity, gender or sexual orientation, political affiliation, religious belief, or other common characteristic. Many of the insults and threats targeted at a group correspond to what is commonly understood as hate speech.\n\nOther (OTH): The target of these offensive posts does not belong to any of the previous two categories (e.g. an organization, a situation, an event, or an issue)." + ] + }, + "annotation_id": "0ce9451170a61260e3458c996de322c4d6caf1f6", + "worker_id": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "Targeted Insult (TIN): Posts which contain an insult/threat to an individual, group, or others ", + "Untargeted (UNT): Posts containing non-targeted profanity and swearing." + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "Level B categorizes the type of offense and two labels are used: targeted (TIN) and untargeted (INT) insults and threats.", + "Targeted Insult (TIN): Posts which contain an insult/threat to an individual, group, or others (see next layer);", + "Untargeted (UNT): Posts containing non-targeted profanity and swearing. Posts with general profanity are not targeted, but they contain non-acceptable language." + ], + "highlighted_evidence": [ + "Level B categorizes the type of offense and two labels are used: targeted (TIN) and untargeted (INT) insults and threats.\n\nTargeted Insult (TIN): Posts which contain an insult/threat to an individual, group, or others (see next layer);\n\nUntargeted (UNT): Posts containing non-targeted profanity and swearing. Posts with general profanity are not targeted, but they contain non-acceptable language." + ] + }, + "annotation_id": "f52f80b4b459acf13024e484953521ce927c3b24", + "worker_id": "34c35a1877e453ecaebcf625df3ef788e1953cc4" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "offensive (OFF) and non-offensive (NOT)", + "targeted (TIN) and untargeted (INT) insults", + "targets of insults and threats as individual (IND), group (GRP), and other (OTH)" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "In the OLID dataset, we use a hierarchical annotation model split into three levels to distinguish between whether language is offensive or not (A), and type (B) and target (C) of the offensive language. Each level is described in more detail in the following subsections and examples are shown in Table TABREF10 .", + "Level A discriminates between offensive (OFF) and non-offensive (NOT) tweets.", + "Level B categorizes the type of offense and two labels are used: targeted (TIN) and untargeted (INT) insults and threats.", + "Level C categorizes the targets of insults and threats as individual (IND), group (GRP), and other (OTH)." + ], + "highlighted_evidence": [ + "In the OLID dataset, we use a hierarchical annotation model split into three levels to distinguish between whether language is offensive or not (A), and type (B) and target (C) of the offensive language.", + "Level A discriminates between offensive (OFF) and non-offensive (NOT) tweets.", + "Level B categorizes the type of offense and two labels are used: targeted (TIN) and untargeted (INT) insults and threats.", + "Level C categorizes the targets of insults and threats as individual (IND), group (GRP), and other (OTH)." + ] + }, + "annotation_id": "f6524c3b1b2294e6e8dffdb005192dad12b30dd8", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + } + ] + }, + { + "question": "What is the best performing model?", + "question_id": "55bd59076a49b19d3283af41c5e3ccb875f3eb0c", + "nlp_background": "five", + "topic_background": "research", + "paper_read": "yes", + "search_query": "Offensive language", + "question_writer": "5053f146237e8fc8859ed3984b5d3f02f39266b7", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "CNN " + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "The performance on discriminating between offensive (OFF) and non-offensive (NOT) posts is reported in Table TABREF18 . We can see that all systems perform significantly better than chance, with the neural models being substantially better than the SVM. The CNN outperforms the RNN model, achieving a macro-F1 score of 0.80.", + "The CNN system achieved higher performance in this experiment compared to the BiLSTM, with a macro-F1 score of 0.69. All systems performed better at identifying target and threats (TIN) than untargeted offenses (UNT)." + ], + "highlighted_evidence": [ + "The performance on discriminating between offensive (OFF) and non-offensive (NOT) posts is reported in Table TABREF18 . We can see that all systems perform significantly better than chance, with the neural models being substantially better than the SVM. The CNN outperforms the RNN model, achieving a macro-F1 score of 0.80.", + "The CNN system achieved higher performance in this experiment compared to the BiLSTM, with a macro-F1 score of 0.69. All systems performed better at identifying target and threats (TIN) than untargeted offenses (UNT)." + ] + }, + "annotation_id": "cba16ebea4215ad9e9fe5250a3ba2d46f265f40d", + "worker_id": "a0b403873302db7cada39008f04d01155ef68f4f" + } + ] + }, + { + "question": "How many annotators participated?", + "question_id": "521280a87c43fcdf9f577da235e7072a23f0673e", + "nlp_background": "five", + "topic_background": "research", + "paper_read": "yes", + "search_query": "Offensive language", + "question_writer": "5053f146237e8fc8859ed3984b5d3f02f39266b7", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "five annotators" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "The data included in OLID has been collected from Twitter. We retrieved the data using the Twitter API by searching for keywords and constructions that are often included in offensive messages, such as `she is' or `to:BreitBartNews'. We carried out a first round of trial annotation of 300 instances with six experts. The goal of the trial annotation was to 1) evaluate the proposed tagset; 2) evaluate the data retrieval method; and 3) create a gold standard with instances that could be used as test questions in the training and test setting annotation which was carried out using crowdsourcing. The breakdown of keywords and their offensive content in the trial data of 300 tweets is shown in Table TABREF14 . We included a left (@NewYorker) and far-right (@BreitBartNews) news accounts because there tends to be political offense in the comments. One of the best offensive keywords was tweets that were flagged as not being safe by the Twitter `safe' filter (the `-' indicates `not safe'). The vast majority of content on Twitter is not offensive so we tried different strategies to keep a reasonable number of tweets in the offensive class amounting to around 30% of the dataset including excluding some keywords that were not high in offensive content such as `they are` and `to:NewYorker`. Although `he is' is lower in offensive content we kept it as a keyword to avoid gender bias. In addition to the keywords in the trial set, we searched for more political keywords which tend to be higher in offensive content, and sampled our dataset such that 50% of the the tweets come from political keywords and 50% come from non-political keywords. In addition to the keywords `gun control', and `to:BreitbartNews', political keywords used to collect these tweets are `MAGA', `antifa', `conservative' and `liberal'. We computed Fliess' INLINEFORM0 on the trial set for the five annotators on 21 of the tweets. INLINEFORM1 is .83 for Layer A (OFF vs NOT) indicating high agreement. As to normalization and anonymization, no user metadata or Twitter IDs have been stored, and URLs and Twitter mentions have been substituted to placeholders. We follow prior work in related areas (burnap2015cyber,davidson2017automated) and annotate our data using crowdsourcing using the platform Figure Eight. We ensure data quality by: 1) we only received annotations from individuals who were experienced in the platform; and 2) we used test questions to discard annotations of individuals who did not reach a certain threshold. Each instance in the dataset was annotated by multiple annotators and inter-annotator agreement has been calculated. We first acquired two annotations for each instance. In case of 100% agreement, we considered these as acceptable annotations, and in case of disagreement, we requested more annotations until the agreement was above 66%. After the crowdsourcing annotation, we used expert adjudication to guarantee the quality of the annotation. The breakdown of the data into training and testing for the labels from each level is shown in Table TABREF15 ." + ], + "highlighted_evidence": [ + "We computed Fliess' INLINEFORM0 on the trial set for the five annotators on 21 of the tweets. INLINEFORM1 is .83 for Layer A (OFF vs NOT) indicating high agreement. As to normalization and anonymization, no user metadata or Twitter IDs have been stored, and URLs and Twitter mentions have been substituted to placeholders. " + ] + }, + "annotation_id": "4001453e026a63ebbce59e23f86431dad1a12d37", + "worker_id": "a0b403873302db7cada39008f04d01155ef68f4f" + } + ] + }, + { + "question": "What is the definition of offensive language?", + "question_id": "5a8cc8f80509ea77d8213ed28c5ead501c68c725", + "nlp_background": "five", + "topic_background": "research", + "paper_read": "yes", + "search_query": "Offensive language", + "question_writer": "5053f146237e8fc8859ed3984b5d3f02f39266b7", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + " Most prior work focuses on a different aspect of offensive language such as abusive language BIBREF0 , BIBREF1 , (cyber-)aggression BIBREF2 , (cyber-)bullying BIBREF3 , BIBREF4 , toxic comments INLINEFORM0 , hate speech BIBREF5 , BIBREF6 , BIBREF7 , BIBREF8 , BIBREF9 , BIBREF10 , and offensive language BIBREF11 . Prior work has focused on these aspects of offensive language in Twitter BIBREF3 , BIBREF7 , BIBREF8 , BIBREF11 , Wikipedia comments, and Facebook posts BIBREF2 ." + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "Offensive content has become pervasive in social media and a reason of concern for government organizations, online communities, and social media platforms. One of the most common strategies to tackle the problem is to train systems capable of recognizing offensive content, which then can be deleted or set aside for human moderation. In the last few years, there have been several studies published on the application of computational methods to deal with this problem. Most prior work focuses on a different aspect of offensive language such as abusive language BIBREF0 , BIBREF1 , (cyber-)aggression BIBREF2 , (cyber-)bullying BIBREF3 , BIBREF4 , toxic comments INLINEFORM0 , hate speech BIBREF5 , BIBREF6 , BIBREF7 , BIBREF8 , BIBREF9 , BIBREF10 , and offensive language BIBREF11 . Prior work has focused on these aspects of offensive language in Twitter BIBREF3 , BIBREF7 , BIBREF8 , BIBREF11 , Wikipedia comments, and Facebook posts BIBREF2 ." + ], + "highlighted_evidence": [ + "Most prior work focuses on a different aspect of offensive language such as abusive language BIBREF0 , BIBREF1 , (cyber-)aggression BIBREF2 , (cyber-)bullying BIBREF3 , BIBREF4 , toxic comments INLINEFORM0 , hate speech BIBREF5 , BIBREF6 , BIBREF7 , BIBREF8 , BIBREF9 , BIBREF10 , and offensive language BIBREF11 . Prior work has focused on these aspects of offensive language in Twitter BIBREF3 , BIBREF7 , BIBREF8 , BIBREF11 , Wikipedia comments, and Facebook posts BIBREF2 ." + ] + }, + "annotation_id": "8caf66b3081c2546f5b6b4549641f10ef2f62d77", + "worker_id": "a0b403873302db7cada39008f04d01155ef68f4f" + } + ] + }, + { + "question": "What are the three layers of the annotation scheme?", + "question_id": "290ee79b5e3872e0496a6a0fc9b103ab7d8f6c30", + "nlp_background": "five", + "topic_background": "research", + "paper_read": "yes", + "search_query": "Offensive language", + "question_writer": "5053f146237e8fc8859ed3984b5d3f02f39266b7", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "Level A: Offensive language Detection\n", + "Level B: Categorization of Offensive Language\n", + "Level C: Offensive Language Target Identification\n" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "In the OLID dataset, we use a hierarchical annotation model split into three levels to distinguish between whether language is offensive or not (A), and type (B) and target (C) of the offensive language. Each level is described in more detail in the following subsections and examples are shown in Table TABREF10 .", + "Level A: Offensive language Detection", + "Level A discriminates between offensive (OFF) and non-offensive (NOT) tweets.", + "Level B: Categorization of Offensive Language", + "Level B categorizes the type of offense and two labels are used: targeted (TIN) and untargeted (INT) insults and threats.", + "Level C: Offensive Language Target Identification", + "Level C categorizes the targets of insults and threats as individual (IND), group (GRP), and other (OTH)." + ], + "highlighted_evidence": [ + "n the OLID dataset, we use a hierarchical annotation model split into three levels to distinguish between whether language is offensive or not (A), and type (B) and target (C) of the offensive language. Each level is described in more detail in the following subsections and examples are shown in Table TABREF10 .", + "Level A: Offensive language Detection\nLevel A discriminates between offensive (OFF) and non-offensive (NOT) tweets.", + "Level B: Categorization of Offensive Language\nLevel B categorizes the type of offense and two labels are used: targeted (TIN) and untargeted (INT) insults and threats.", + "Level C: Offensive Language Target Identification\nLevel C categorizes the targets of insults and threats as individual (IND), group (GRP), and other (OTH)." + ] + }, + "annotation_id": "022a7c6e964a11e13c4a1c1a4d10472bff21e480", + "worker_id": "a0b403873302db7cada39008f04d01155ef68f4f" + } + ] + }, + { + "question": "How long is the dataset for each step of hierarchy?", + "question_id": "1b72aa2ec3ce02131e60626639f0cf2056ec23ca", + "nlp_background": "five", + "topic_background": "research", + "paper_read": "yes", + "search_query": "Offensive language", + "question_writer": "5053f146237e8fc8859ed3984b5d3f02f39266b7", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "Level A: 14100 Tweets\nLevel B: 4640 Tweets\nLevel C: 4089 Tweets", + "evidence": [ + "The data included in OLID has been collected from Twitter. We retrieved the data using the Twitter API by searching for keywords and constructions that are often included in offensive messages, such as `she is' or `to:BreitBartNews'. We carried out a first round of trial annotation of 300 instances with six experts. The goal of the trial annotation was to 1) evaluate the proposed tagset; 2) evaluate the data retrieval method; and 3) create a gold standard with instances that could be used as test questions in the training and test setting annotation which was carried out using crowdsourcing. The breakdown of keywords and their offensive content in the trial data of 300 tweets is shown in Table TABREF14 . We included a left (@NewYorker) and far-right (@BreitBartNews) news accounts because there tends to be political offense in the comments. One of the best offensive keywords was tweets that were flagged as not being safe by the Twitter `safe' filter (the `-' indicates `not safe'). The vast majority of content on Twitter is not offensive so we tried different strategies to keep a reasonable number of tweets in the offensive class amounting to around 30% of the dataset including excluding some keywords that were not high in offensive content such as `they are` and `to:NewYorker`. Although `he is' is lower in offensive content we kept it as a keyword to avoid gender bias. In addition to the keywords in the trial set, we searched for more political keywords which tend to be higher in offensive content, and sampled our dataset such that 50% of the the tweets come from political keywords and 50% come from non-political keywords. In addition to the keywords `gun control', and `to:BreitbartNews', political keywords used to collect these tweets are `MAGA', `antifa', `conservative' and `liberal'. We computed Fliess' INLINEFORM0 on the trial set for the five annotators on 21 of the tweets. INLINEFORM1 is .83 for Layer A (OFF vs NOT) indicating high agreement. As to normalization and anonymization, no user metadata or Twitter IDs have been stored, and URLs and Twitter mentions have been substituted to placeholders. We follow prior work in related areas (burnap2015cyber,davidson2017automated) and annotate our data using crowdsourcing using the platform Figure Eight. We ensure data quality by: 1) we only received annotations from individuals who were experienced in the platform; and 2) we used test questions to discard annotations of individuals who did not reach a certain threshold. Each instance in the dataset was annotated by multiple annotators and inter-annotator agreement has been calculated. We first acquired two annotations for each instance. In case of 100% agreement, we considered these as acceptable annotations, and in case of disagreement, we requested more annotations until the agreement was above 66%. After the crowdsourcing annotation, we used expert adjudication to guarantee the quality of the annotation. The breakdown of the data into training and testing for the labels from each level is shown in Table TABREF15 .", + "FLOAT SELECTED: Table 3: Distribution of label combinations in OLID." + ], + "highlighted_evidence": [ + " The breakdown of the data into training and testing for the labels from each level is shown in Table TABREF15 .", + "FLOAT SELECTED: Table 3: Distribution of label combinations in OLID." + ] + }, + "annotation_id": "1ef414648e053b756ab33357e5172602e9eccaae", + "worker_id": "a0b403873302db7cada39008f04d01155ef68f4f" + } + ] + } + ], + "figures_and_tables": [ + { + "file": "2-Table1-1.png", + "caption": "Table 1: Four tweets from the dataset, with their labels for each level of the annotation schema." + }, + { + "file": "3-Table2-1.png", + "caption": "Table 2: The keywords from the full dataset (except for the first three rows) and the percentage of offensive tweets for each keyword." + }, + { + "file": "4-Table3-1.png", + "caption": "Table 3: Distribution of label combinations in OLID." + }, + { + "file": "5-Table4-1.png", + "caption": "Table 4: Results for offensive language detection (Level A). We report Precision (P), Recall (R), and F1 for each model/baseline on all classes (NOT, OFF), and weighted averages. Macro-F1 is also listed (best in bold)." + }, + { + "file": "5-Table5-1.png", + "caption": "Table 5: Results for offensive language categorization (level B). We report Precision (P), Recall (R), and F1 for each model/baseline on all classes (TIN, UNT), and weighted averages. Macro-F1 is also listed (best in bold)." + }, + { + "file": "5-Table6-1.png", + "caption": "Table 6: Results for offense target identification (level C). We report Precision (P), Recall (R), and F1 for each model/baseline on all classes (GRP, IND, OTH), and weighted averages. Macro-F1 is also listed (best in bold)." + } + ] + }, + "2001.01269": { + "title": "Generating Word and Document Embeddings for Sentiment Analysis", + "abstract": "Sentiments of words differ from one corpus to another. Inducing general sentiment lexicons for languages and using them cannot, in general, produce meaningful results for different domains. In this paper, we combine contextual and supervised information with the general semantic representations of words occurring in the dictionary. Contexts of words help us capture the domain-specific information and supervised scores of words are indicative of the polarities of those words. When we combine supervised features of words with the features extracted from their dictionary definitions, we observe an increase in the success rates. We try out the combinations of contextual, supervised, and dictionary-based approaches, and generate original vectors. We also combine the word2vec approach with hand-crafted features. We induce domain-specific sentimental vectors for two corpora, which are the movie domain and the Twitter datasets in Turkish. When we thereafter generate document vectors and employ the support vector machines method utilising those vectors, our approaches perform better than the baseline studies for Turkish with a significant margin. We evaluated our models on two English corpora as well and these also outperformed the word2vec approach. It shows that our approaches are cross-lingual and cross-domain.", + "full_text": [ + { + "section_name": "Introduction", + "paragraphs": [ + "Sentiment analysis has recently been one of the hottest topics in natural language processing (NLP). It is used to identify and categorise opinions expressed by reviewers on a topic or an entity. Sentiment analysis can be leveraged in marketing, social media analysis, and customer service. Although many studies have been conducted for sentiment analysis in widely spoken languages, this topic is still immature for Turkish and many other languages.", + "Neural networks outperform the conventional machine learning algorithms in most classification tasks, including sentiment analysis BIBREF0. In these networks, word embedding vectors are fed as input to overcome the data sparsity problem and make the representations of words more “meaningful” and robust. Those embeddings indicate how close the words are to each other in the vector space model (VSM).", + "Most of the studies utilise embeddings, such as word2vec BIBREF1, which take into account the syntactic and semantic representations of the words only. Discarding the sentimental aspects of words may lead to words of different polarities being close to each other in the VSM, if they share similar semantic and syntactic features.", + "For Turkish, there are only a few studies which leverage sentimental information in generating the word and document embeddings. Unlike the studies conducted for English and other widely-spoken languages, in this paper, we use the official dictionaries for this language and combine the unsupervised and supervised scores to generate a unified score for each dimension of the word embeddings in this task.", + "Our main contribution is to create original and effective word vectors that capture syntactic, semantic, and sentimental characteristics of words, and use all of this knowledge in generating embeddings. We also utilise the word2vec embeddings trained on a large corpus. Besides using these word embeddings, we also generate hand-crafted features on a review basis and create document vectors. We evaluate those embeddings on two datasets. The results show that we outperform the approaches which do not take into account the sentimental information. We also had better performances than other studies carried out on sentiment analysis in Turkish media. We also evaluated our novel embedding approaches on two English corpora of different genres. We outperformed the baseline approaches for this language as well. The source code and datasets are publicly available.", + "The paper is organised as follows. In Section 2, we present the existing works on sentiment classification. In Section 3, we describe the methods proposed in this work. The experimental results are shown and the main contributions of our proposed approach are discussed in Section 4. In Section 5, we conclude the paper." + ] + }, + { + "section_name": "Related Work", + "paragraphs": [ + "In the literature, the main consensus is that the use of dense word embeddings outperform the sparse embeddings in many tasks. Latent semantic analysis (LSA) used to be the most popular method in generating word embeddings before the invention of the word2vec and other word vector algorithms which are mostly created by shallow neural network models. Although many studies have been employed on generating word vectors including both semantic and sentimental components, generating and analysing the effects of different types of embeddings on different tasks is an emerging field for Turkish.", + "Latent Dirichlet allocation (LDA) is used in BIBREF2 to extract mixture of latent topics. However, it focusses on finding the latent topics of a document, not the word meanings themselves. In BIBREF3, LSA is utilised to generate word vectors, leveraging indirect co-occurrence statistics. These outperform the use of sparse vectors BIBREF4.", + "Some of the prior studies have also taken into account the sentimental characteristics of a word when creating word vectors BIBREF5, BIBREF6, BIBREF7. A model with semantic and sentiment components is built in BIBREF8, making use of star-ratings of reviews. In BIBREF9, a sentiment lexicon is induced preferring the use of domain-specific co-occurrence statistics over the word2vec method and they outperform the latter.", + "In a recent work on sentiment analysis in Turkish BIBREF10, they learn embeddings using Turkish social media. They use the word2vec algorithm, create several unsupervised hand-crafted features, generate document vectors and feed them as input into the support vector machines (SVM) approach. We outperform this baseline approach using more effective word embeddings and supervised hand-crafted features.", + "In English, much of the recent work on learning sentiment-specific embeddings relies only on distant supervision. In BIBREF11, emojis are used as features and a bi-LSTM (bidirectional long short-term memory) neural network model is built to learn sentiment-aware word embeddings. In BIBREF12, a neural network that learns word embeddings is built by using contextual information about the data and supervised scores of the words. This work captures the supervised information by utilising emoticons as features. Most of our approaches do not rely on a neural network model in learning embeddings. However, they produce state-of-the-art results." + ] + }, + { + "section_name": "Methodology", + "paragraphs": [ + "We generate several word vectors, which capture the sentimental, lexical, and contextual characteristics of words. In addition to these mostly original vectors, we also create word2vec embeddings to represent the corpus words by training the embedding model on these datasets. After generating these, we combine them with hand-crafted features to create document vectors and perform classification, as will be explained in Section 3.5." + ] + }, + { + "section_name": "Methodology ::: Corpus-based Approach", + "paragraphs": [ + "Contextual information is informative in the sense that, in general, similar words tend to appear in the same contexts. For example, the word smart is more likely to cooccur with the word hardworking than with the word lazy. This similarity can be defined semantically and sentimentally. In the corpus-based approach, we capture both of these characteristics and generate word embeddings specific to a domain.", + "Firstly, we construct a matrix whose entries correspond to the number of cooccurrences of the row and column words in sliding windows. Diagonal entries are assigned the number of sliding windows that the corresponding row word appears in the whole corpus. We then normalise each row by dividing entries in the row by the maximum score in it.", + "Secondly, we perform the principal component analysis (PCA) method to reduce the dimensionality. It captures latent meanings and takes into account high-order cooccurrence removing noise. The attribute (column) number of the matrix is reduced to 200. We then compute cosine similarity between each row pair $w_i$ and $w_j$ as in (DISPLAY_FORM3) to find out how similar two word vectors (rows) are.", + "Thirdly, all the values in the matrix are subtracted from 1 to create a dissimilarity matrix. Then, we feed the matrix as input into the fuzzy c-means clustering algorithm. We chose the number of clusters as 200, as it is considered a standard for word embeddings in the literature. After clustering, the dimension i for a corresponding word indicates the degree to which this word belongs to cluster i. The intuition behind this idea is that if two words are similar in the VSM, they are more likely to belong to the same clusters with akin probabilities. In the end, each word in the corpus is represented by a 200-dimensional vector.", + "In addition to this method, we also perform singular value decomposition (SVD) on the cooccurrence matrices, where we compute the matrix $M^{PPMI} = U\\Sigma V^{T}$. Positive pointwise mutual information (PPMI) scores between words are calculated and the truncated singular value decomposition is computed. We take into account the U matrix only for each word. We have chosen the singular value number as 200. That is, each word in the corpus is represented by a 200-dimensional vector as follows." + ] + }, + { + "section_name": "Methodology ::: Dictionary-based Approach", + "paragraphs": [ + "In Turkish, there do not exist well-established sentiment lexicons as in English. In this approach, we made use of the TDK (Türk Dil Kurumu - “Turkish Language Institution”) dictionary to obtain word polarities. Although it is not a sentiment lexicon, combining it with domain-specific polarity scores obtained from the corpus led us to have state-of-the-art results.", + "We first construct a matrix whose row entries are corpus words and column entries are the words in their dictionary definitions. We followed the boolean approach. For instance, for the word cat, the column words occurring in its dictionary definition are given a score of 1. Those column words not appearing in the definition of cat are assigned a score of 0 for that corresponding row entry.", + "When we performed clustering on this matrix, we observed that those words having similar meanings are, in general, assigned to the same clusters. However, this similarity fails in capturing the sentimental characteristics. For instance, the words happy and unhappy are assigned to the same cluster, since they have the same words, such as feeling, in their dictionary definitions. However, they are of opposite polarities and should be discerned from each other.", + "Therefore, we utilise a metric to move such words away from each other in the VSM, even though they have common words in their dictionary definitions. We multiply each value in a row with the corresponding row word's raw supervised score, thereby having more meaningful clusters. Using the training data only, the supervised polarity score per word is calculated as in (DISPLAY_FORM4).", + "Here, $ w_{t}$ denotes the sentiment score of word $t$, $N_{t}$ is the number of documents (reviews or tweets) in which $t$ occurs in the dataset of positive polarity, $N$ is the number of all the words in the corpus of positive polarity. $N^{\\prime }$ denotes the corpus of negative polarity. $N^{\\prime }_{t}$ and $N^{\\prime }$ denote similar values for the negative polarity corpus. We perform normalisation to prevent the imbalance problem and add a small number to both numerator and denominator for smoothing.", + "As an alternative to multiplying with the supervised polarity scores, we also separately multiplied all the row scores with only +1 if the row word is a positive word, and with -1 if it is a negative word. We have observed it boosts the performance more compared to using raw scores.", + "The effect of this multiplication is exemplified in Figure FIGREF7, showing the positions of word vectors in the VSM. Those “x\" words are sentimentally negative words, those “o\" words are sentimentally positive ones. On the top coordinate plane, the words of opposite polarities are found to be close to each other, since they have common words in their dictionary definitions. Only the information concerned with the dictionary definitions are used there, discarding the polarity scores. However, when we utilise the supervised score (+1 or -1), words of opposite polarities (e.g. “happy\" and “unhappy\") get far away from each other as they are translated across coordinate regions. Positive words now appear in quadrant 1, whereas negative words appear in quadrant 3. Thus, in the VSM, words that are sentimentally similar to each other could be clustered more accurately. Besides clustering, we also employed the SVD method to perform dimensionality reduction on the unsupervised dictionary algorithm and used the newly generated matrix by combining it with other subapproaches. The number of dimensions is chosen as 200 again according to the $U$ matrix. The details are given in Section 3.4. When using and evaluating this subapproach on the English corpora, we used the SentiWordNet lexicon BIBREF13. We have achieved better results for the dictionary-based algorithm when we employed the SVD reduction method compared to the use of clustering." + ] + }, + { + "section_name": "Methodology ::: Supervised Contextual 4-scores", + "paragraphs": [ + "Our last component is a simple metric that uses four supervised scores for each word in the corpus. We extract these scores as follows. For a target word in the corpus, we scan through all of its contexts. In addition to the target word's polarity score (the self score), out of all the polarity scores of words occurring in the same contexts as the target word, minimum, maximum, and average scores are taken into consideration. The word polarity scores are computed using (DISPLAY_FORM4). Here, we obtain those scores from the training data.", + "The intuition behind this method is that those four scores are more indicative of a word's polarity rather than only one (the self score). This approach is fully supervised unlike the previous two approaches." + ] + }, + { + "section_name": "Methodology ::: Combination of the Word Embeddings", + "paragraphs": [ + "In addition to using the three approaches independently, we also combined all the matrices generated in the previous approaches. That is, we concatenate the reduced forms (SVD - U) of corpus-based, dictionary-based, and the whole of 4-score vectors of each word, horizontally. Accordingly, each corpus word is represented by a 404-dimensional vector, since corpus-based and dictionary-based vector components are each composed of 200 dimensions, whereas the 4-score vector component is formed by four values.", + "The main intuition behind the ensemble method is that some approaches compensate for what the others may lack. For example, the corpus-based approach captures the domain-specific, semantic, and syntactic characteristics. On the other hand, the 4-scores method captures supervised features, and the dictionary-based approach is helpful in capturing the general semantic characteristics. That is, combining those three approaches makes word vectors more representative." + ] + }, + { + "section_name": "Methodology ::: Generating Document Vectors", + "paragraphs": [ + "After creating several embeddings as mentioned above, we create document (review or tweet) vectors. For each document, we sum all the vectors of words occurring in that document and take their average. In addition to it, we extract three hand-crafted polarity scores, which are minimum, mean, and maximum polarity scores, from each review. These polarity scores of words are computed as in (DISPLAY_FORM4). For example, if a review consists of five words, it would have five polarity scores and we utilise only three of these sentiment scores as mentioned. Lastly, we concatenate these three scores to the averaged word vector per review.", + "That is, each review is represented by the average word vector of its constituent word embeddings and three supervised scores. We then feed these inputs into the SVM approach. The flowchart of our framework is given in Figure FIGREF11. When combining the unsupervised features, which are word vectors created on a word basis, with supervised three scores extracted on a review basis, we have better state-of-the-art results." + ] + }, + { + "section_name": "Datasets", + "paragraphs": [ + "We utilised two datasets for both Turkish and English to evaluate our methods.", + "For Turkish, as the first dataset, we utilised the movie reviews which are collected from a popular website. The number of reviews in this movie corpus is 20,244 and the average number of words in reviews is 39. Each of these reviews has a star-rating score which is indicative of sentiment. These polarity scores are between the values 0.5 and 5, at intervals of 0.5. We consider a review to be negative it the score is equal to or lower than 2.5. On the other hand, if it is equal to or higher than 4, it is assumed to be positive. We have randomly selected 7,020 negative and 7,020 positive reviews and processed only them.", + "The second Turkish dataset is the Twitter corpus which is formed of tweets about Turkish mobile network operators. Those tweets are mostly much noisier and shorter compared to the reviews in the movie corpus. In total, there are 1,716 tweets. 973 of them are negative and 743 of them are positive. These tweets are manually annotated by two humans, where the labels are either positive or negative. We measured the Cohen's Kappa inter-annotator agreement score to be 0.82. If there was a disagreement on the polarity of a tweet, we removed it.", + "We also utilised two other datasets in English to test the cross-linguality of our approaches. One of them is a movie corpus collected from the web. There are 5,331 positive reviews and 5,331 negative reviews in this corpus. The other is a Twitter dataset, which has nearly 1.6 million tweets annotated through a distant supervised method BIBREF14. These tweets have positive, neutral, and negative labels. We have selected 7,020 positive tweets and 7,020 negative tweets randomly to generate a balanced dataset." + ] + }, + { + "section_name": "Experiments ::: Preprocessing", + "paragraphs": [ + "In Turkish, people sometimes prefer to spell English characters for the corresponding Turkish characters (e.g. i for ı, c for ç) when writing in electronic format. To normalise such words, we used the Zemberek tool BIBREF15. All punctuation marks except “!\" and “?\" are removed, since they do not contribute much to the polarity of a document. We took into account emoticons, such as “:))\", and idioms, such as “kafayı yemek” (lose one's mind), since two or more words can express a sentiment together, irrespective of the individual words thereof. Since Turkish is an agglutinative language, we used the morphological parser and disambiguator tools BIBREF16, BIBREF17. We also performed negation handling and stop-word elimination. In negation handling, we append an underscore to the end of a word if it is negated. For example, “güzel değil\" (not beautiful) is redefined as “güzel_\" (beautiful_) in the feature selection stage when supervised scores are being computed." + ] + }, + { + "section_name": "Experiments ::: Hyperparameters", + "paragraphs": [ + "We used the LibSVM utility of the WEKA tool. We chose the linear kernel option to classify the reviews. We trained word2vec embeddings on all the four corpora using the Gensim library BIBREF18 with the skip-gram method. The dimension size of these embeddings are set at 200. As mentioned, other embeddings, which are generated utilising the clustering and the SVD approach, are also of size 200. For c-mean clustering, we set the maximum number of iterations at 25, unless it converges." + ] + }, + { + "section_name": "Experiments ::: Results", + "paragraphs": [ + "We evaluated our models on four corpora, which are the movie and the Twitter datasets in Turkish and English. All of the embeddings are learnt on four corpora separately. We have used the accuracy metric since all the datasets are completely or nearly completely balanced. We performed 10-fold cross-validation for both of the datasets. We used the approximate randomisation technique to test whether our results are statistically significant. Here, we tried to predict the labels of reviews and assess the performance.", + "We obtained varying accuracies as shown in Table TABREF17. “3 feats\" features are those hand-crafted features we extracted, which are the minimum, mean, and maximum polarity scores of the reviews as explained in Section 3.5. As can be seen, at least one of our methods outperforms the baseline word2vec approach for all the Turkish and English corpora, and all categories. All of our approaches performed better when we used the supervised scores, which are extracted on a review basis, and concatenated them to word vectors. Mostly, the supervised 4-scores feature leads to the highest accuracies, since it employs the annotational information concerned with polarities on a word basis.", + "As can be seen in Table TABREF17, the clustering method, in general, yields the lowest scores. We found out that the corpus - SVD metric does always perform better than the clustering method. We attribute it to that in SVD the most important singular values are taken into account. The corpus - SVD technique outperforms the word2vec algorithm for some corpora. When we do not take into account the 3-feats technique, the corpus-based SVD method yields the highest accuracies for the English Twitter dataset. We show that simple models can outperform more complex models, such as the concatenation of the three subapproaches or the word2vec algorithm. Another interesting finding is that for some cases the accuracy decreases when we utilise the polarity labels, as in the case for the English Twitter dataset.", + "Since the TDK dictionary covers most of the domain-specific vocabulary used in the movie reviews, the dictionary method performs well. However, the dictionary lacks many of the words, occurring in the tweets; therefore, its performance is not the best of all. When the TDK method is combined with the 3-feats technique, we observed a great improvement, as can be expected.", + "Success rates obtained for the movie corpus are much better than those for the Twitter dataset for most of our approaches, since tweets are, in general, much shorter and noisier. We also found out that, when choosing the p value as 0.05, our results are statistically significant compared to the baseline approach in Turkish BIBREF10. Some of our subapproaches also produce better success rates than those sentiment analysis models employed in English BIBREF11, BIBREF12. We have achieved state-of-the-art results for the sentiment classification task for both Turkish and English. As mentioned, our approaches, in general, perform best in predicting the labels of reviews when three supervised scores are additionality utilised.", + "We also employed the convolutional neural network model (CNN). However, the SVM classifier, which is a conventional machine learning algorithm, performed better. We did not include the performances of CNN for embedding types here due to the page limit of the paper.", + "As a qualitative assessment of the word representations, given some query words we visualised the most similar words to those words using the cosine similarity metric. By assessing the similarities between a word and all the other corpus words, we can find the most akin words according to different approaches. Table TABREF18 shows the most similar words to given query words. Those words which are indicative of sentiment are, in general, found to be most similar to those words of the same polarity. For example, the most akin word to muhteşem (gorgeous) is 10/10, both of which have positive polarity. As can be seen in Table TABREF18, our corpus-based approach is more adept at capturing domain-specific features as compared to word2vec, which generally captures general semantic and syntactic characteristics, but not the sentimental ones." + ] + }, + { + "section_name": "Conclusion", + "paragraphs": [ + "We have demonstrated that using word vectors that capture only semantic and syntactic characteristics may be improved by taking into account their sentimental aspects as well. Our approaches are cross-lingual and cross-domain. They can be applied to other domains and other languages than Turkish and English with minor changes.", + "Our study is one of the few ones that perform sentiment analysis in Turkish and leverages sentimental characteristics of words in generating word vectors and outperforms all the others. Any of the approaches we propose can be used independently of the others. Our approaches without using sentiment labels can be applied to other classification tasks, such as topic classification and concept mining.", + "The experiments show that even unsupervised approaches, as in the corpus-based approach, can outperform supervised approaches in classification tasks. Combining some approaches, which can compensate for what others lack, can help us build better vectors. Our word vectors are created by conventional machine learning algorithms; however, they, as in the corpus-based model, produce state-of-the-art results. Although we preferred to use a classical machine learning algorithm, which is SVM, over a neural network classifier to predict the labels of reviews, we achieved accuracies of over 90 per cent for the Turkish movie corpus and about 88 per cent for the English Twitter dataset.", + "We performed only binary sentiment classification in this study as most of the studies in the literature do. We will extend our system in future by using neutral reviews as well. We also plan to employ Turkish WordNet to enhance the generalisability of our embeddings as another future work." + ] + }, + { + "section_name": "Acknowledgments", + "paragraphs": [ + "This work was supported by Boğaziçi University Research Fund Grant Number 6980D, and by Turkish Ministry of Development under the TAM Project number DPT2007K12-0610. Cem Rıfkı Aydın is supported by TÜBİTAK BIDEB 2211E." + ] + } + ], + "qas": [ + { + "question": "What baseline method is used?", + "question_id": "81d193672090295e687bc4f4ac1b7a9c76ea35df", + "nlp_background": "five", + "topic_background": "unfamiliar", + "paper_read": "no", + "search_query": "twitter", + "question_writer": "e8b24c3133e0bec0a6465e1f13acd3a5ed816b37", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "using word2vec to create features that are used as input to the SVM", + "evidence": [ + "In a recent work on sentiment analysis in Turkish BIBREF10, they learn embeddings using Turkish social media. They use the word2vec algorithm, create several unsupervised hand-crafted features, generate document vectors and feed them as input into the support vector machines (SVM) approach. We outperform this baseline approach using more effective word embeddings and supervised hand-crafted features." + ], + "highlighted_evidence": [ + "In a recent work on sentiment analysis in Turkish BIBREF10, they learn embeddings using Turkish social media. They use the word2vec algorithm, create several unsupervised hand-crafted features, generate document vectors and feed them as input into the support vector machines (SVM) approach. We outperform this baseline approach using more effective word embeddings and supervised hand-crafted features." + ] + }, + "annotation_id": "d4096bad9c7a7e8a7e9e37f4456548a44ff63b53", + "worker_id": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "use the word2vec algorithm, create several unsupervised hand-crafted features, generate document vectors and feed them as input into the support vector machines (SVM) approach" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "In a recent work on sentiment analysis in Turkish BIBREF10, they learn embeddings using Turkish social media. They use the word2vec algorithm, create several unsupervised hand-crafted features, generate document vectors and feed them as input into the support vector machines (SVM) approach. We outperform this baseline approach using more effective word embeddings and supervised hand-crafted features." + ], + "highlighted_evidence": [ + "They use the word2vec algorithm, create several unsupervised hand-crafted features, generate document vectors and feed them as input into the support vector machines (SVM) approach." + ] + }, + "annotation_id": "f281d455a8764605b74efd28cd49a64adfd2215a", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + } + ] + }, + { + "question": "What details are given about the Twitter dataset?", + "question_id": "cf171fad0bea5ab985c53d11e48e7883c23cdc44", + "nlp_background": "five", + "topic_background": "unfamiliar", + "paper_read": "no", + "search_query": "twitter", + "question_writer": "e8b24c3133e0bec0a6465e1f13acd3a5ed816b37", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "Those tweets are mostly much noisier and shorter compared to the reviews in the movie corpus. In total, there are 1,716 tweets. 973 of them are negative and 743 of them are positive." + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "The second Turkish dataset is the Twitter corpus which is formed of tweets about Turkish mobile network operators. Those tweets are mostly much noisier and shorter compared to the reviews in the movie corpus. In total, there are 1,716 tweets. 973 of them are negative and 743 of them are positive. These tweets are manually annotated by two humans, where the labels are either positive or negative. We measured the Cohen's Kappa inter-annotator agreement score to be 0.82. If there was a disagreement on the polarity of a tweet, we removed it." + ], + "highlighted_evidence": [ + "The second Turkish dataset is the Twitter corpus which is formed of tweets about Turkish mobile network operators. Those tweets are mostly much noisier and shorter compared to the reviews in the movie corpus. In total, there are 1,716 tweets. 973 of them are negative and 743 of them are positive. These tweets are manually annotated by two humans, where the labels are either positive or negative." + ] + }, + "annotation_id": "83937cb549cc85bae93dd1be666a744a973a7942", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "one of the Twitter datasets is about Turkish mobile network operators, there are positive, neutral and negative labels and provide the total amount plus the distribution of labels", + "evidence": [ + "The second Turkish dataset is the Twitter corpus which is formed of tweets about Turkish mobile network operators. Those tweets are mostly much noisier and shorter compared to the reviews in the movie corpus. In total, there are 1,716 tweets. 973 of them are negative and 743 of them are positive. These tweets are manually annotated by two humans, where the labels are either positive or negative. We measured the Cohen's Kappa inter-annotator agreement score to be 0.82. If there was a disagreement on the polarity of a tweet, we removed it.", + "We also utilised two other datasets in English to test the cross-linguality of our approaches. One of them is a movie corpus collected from the web. There are 5,331 positive reviews and 5,331 negative reviews in this corpus. The other is a Twitter dataset, which has nearly 1.6 million tweets annotated through a distant supervised method BIBREF14. These tweets have positive, neutral, and negative labels. We have selected 7,020 positive tweets and 7,020 negative tweets randomly to generate a balanced dataset." + ], + "highlighted_evidence": [ + "The second Turkish dataset is the Twitter corpus which is formed of tweets about Turkish mobile network operators. Those tweets are mostly much noisier and shorter compared to the reviews in the movie corpus. In total, there are 1,716 tweets. 973 of them are negative and 743 of them are positive. These tweets are manually annotated by two humans, where the labels are either positive or negative. We measured the Cohen's Kappa inter-annotator agreement score to be 0.82. If there was a disagreement on the polarity of a tweet, we removed it.\n\nWe also utilised two other datasets in English to test the cross-linguality of our approaches. One of them is a movie corpus collected from the web. There are 5,331 positive reviews and 5,331 negative reviews in this corpus. The other is a Twitter dataset, which has nearly 1.6 million tweets annotated through a distant supervised method BIBREF14. These tweets have positive, neutral, and negative labels. We have selected 7,020 positive tweets and 7,020 negative tweets randomly to generate a balanced dataset." + ] + }, + "annotation_id": "ee576cbff7447c74aa5d688e28fae68daea1b86a", + "worker_id": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4" + } + ] + }, + { + "question": "What details are given about the movie domain dataset?", + "question_id": "2a564b092916f2fabbfe893cf13de169945ef2e1", + "nlp_background": "five", + "topic_background": "unfamiliar", + "paper_read": "no", + "search_query": "twitter", + "question_writer": "e8b24c3133e0bec0a6465e1f13acd3a5ed816b37", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "there are 20,244 reviews divided into positive and negative with an average 39 words per review, each one having a star-rating score", + "evidence": [ + "For Turkish, as the first dataset, we utilised the movie reviews which are collected from a popular website. The number of reviews in this movie corpus is 20,244 and the average number of words in reviews is 39. Each of these reviews has a star-rating score which is indicative of sentiment. These polarity scores are between the values 0.5 and 5, at intervals of 0.5. We consider a review to be negative it the score is equal to or lower than 2.5. On the other hand, if it is equal to or higher than 4, it is assumed to be positive. We have randomly selected 7,020 negative and 7,020 positive reviews and processed only them." + ], + "highlighted_evidence": [ + "For Turkish, as the first dataset, we utilised the movie reviews which are collected from a popular website. The number of reviews in this movie corpus is 20,244 and the average number of words in reviews is 39. Each of these reviews has a star-rating score which is indicative of sentiment. These polarity scores are between the values 0.5 and 5, at intervals of 0.5. We consider a review to be negative it the score is equal to or lower than 2.5. On the other hand, if it is equal to or higher than 4, it is assumed to be positive. We have randomly selected 7,020 negative and 7,020 positive reviews and processed only them.\n\n" + ] + }, + "annotation_id": "0c776d0d1cbdcbb3f39c2748589aa20ee755adc2", + "worker_id": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "The number of reviews in this movie corpus is 20,244 and the average number of words in reviews is 39. Each of these reviews has a star-rating score which is indicative of sentiment." + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "For Turkish, as the first dataset, we utilised the movie reviews which are collected from a popular website. The number of reviews in this movie corpus is 20,244 and the average number of words in reviews is 39. Each of these reviews has a star-rating score which is indicative of sentiment. These polarity scores are between the values 0.5 and 5, at intervals of 0.5. We consider a review to be negative it the score is equal to or lower than 2.5. On the other hand, if it is equal to or higher than 4, it is assumed to be positive. We have randomly selected 7,020 negative and 7,020 positive reviews and processed only them." + ], + "highlighted_evidence": [ + "For Turkish, as the first dataset, we utilised the movie reviews which are collected from a popular website. The number of reviews in this movie corpus is 20,244 and the average number of words in reviews is 39. Each of these reviews has a star-rating score which is indicative of sentiment." + ] + }, + "annotation_id": "80295b295b55c86e567a782e5d7d15f1f054d434", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + } + ] + }, + { + "question": "Which hand-crafted features are combined with word2vec?", + "question_id": "0d34c0812f1e69ea33f76ca8c24c23b0415ebc8d", + "nlp_background": "five", + "topic_background": "unfamiliar", + "paper_read": "no", + "search_query": "twitter", + "question_writer": "e8b24c3133e0bec0a6465e1f13acd3a5ed816b37", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "three hand-crafted polarity scores, which are minimum, mean, and maximum polarity scores" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "After creating several embeddings as mentioned above, we create document (review or tweet) vectors. For each document, we sum all the vectors of words occurring in that document and take their average. In addition to it, we extract three hand-crafted polarity scores, which are minimum, mean, and maximum polarity scores, from each review. These polarity scores of words are computed as in (DISPLAY_FORM4). For example, if a review consists of five words, it would have five polarity scores and we utilise only three of these sentiment scores as mentioned. Lastly, we concatenate these three scores to the averaged word vector per review.", + "That is, each review is represented by the average word vector of its constituent word embeddings and three supervised scores. We then feed these inputs into the SVM approach. The flowchart of our framework is given in Figure FIGREF11. When combining the unsupervised features, which are word vectors created on a word basis, with supervised three scores extracted on a review basis, we have better state-of-the-art results." + ], + "highlighted_evidence": [ + "In addition to it, we extract three hand-crafted polarity scores, which are minimum, mean, and maximum polarity scores, from each review. These polarity scores of words are computed as in (DISPLAY_FORM4). For example, if a review consists of five words, it would have five polarity scores and we utilise only three of these sentiment scores as mentioned. Lastly, we concatenate these three scores to the averaged word vector per review.\n\nThat is, each review is represented by the average word vector of its constituent word embeddings and three supervised scores. We then feed these inputs into the SVM approach. The flowchart of our framework is given in Figure FIGREF11. When combining the unsupervised features, which are word vectors created on a word basis, with supervised three scores extracted on a review basis, we have better state-of-the-art results." + ] + }, + "annotation_id": "edd9baca583d1a7b29e850a57cadf88c8f463818", + "worker_id": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "polarity scores, which are minimum, mean, and maximum polarity scores, from each review" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "After creating several embeddings as mentioned above, we create document (review or tweet) vectors. For each document, we sum all the vectors of words occurring in that document and take their average. In addition to it, we extract three hand-crafted polarity scores, which are minimum, mean, and maximum polarity scores, from each review. These polarity scores of words are computed as in (DISPLAY_FORM4). For example, if a review consists of five words, it would have five polarity scores and we utilise only three of these sentiment scores as mentioned. Lastly, we concatenate these three scores to the averaged word vector per review." + ], + "highlighted_evidence": [ + "In addition to it, we extract three hand-crafted polarity scores, which are minimum, mean, and maximum polarity scores, from each review." + ] + }, + "annotation_id": "f39d3bb086e25eba1abe85132d0e3fdfaf24cf25", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + } + ] + }, + { + "question": "What word-based and dictionary-based feature are used?", + "question_id": "73e83c54251f6a07744413ac8b8bed6480b2294f", + "nlp_background": "five", + "topic_background": "unfamiliar", + "paper_read": "no", + "search_query": "twitter", + "question_writer": "e8b24c3133e0bec0a6465e1f13acd3a5ed816b37", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "generate word embeddings specific to a domain", + "TDK (Türk Dil Kurumu - “Turkish Language Institution”) dictionary to obtain word polarities" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "Methodology ::: Corpus-based Approach", + "Contextual information is informative in the sense that, in general, similar words tend to appear in the same contexts. For example, the word smart is more likely to cooccur with the word hardworking than with the word lazy. This similarity can be defined semantically and sentimentally. In the corpus-based approach, we capture both of these characteristics and generate word embeddings specific to a domain.", + "Methodology ::: Dictionary-based Approach", + "In Turkish, there do not exist well-established sentiment lexicons as in English. In this approach, we made use of the TDK (Türk Dil Kurumu - “Turkish Language Institution”) dictionary to obtain word polarities. Although it is not a sentiment lexicon, combining it with domain-specific polarity scores obtained from the corpus led us to have state-of-the-art results." + ], + "highlighted_evidence": [ + "Methodology ::: Corpus-based Approach\nContextual information is informative in the sense that, in general, similar words tend to appear in the same contexts.", + " In the corpus-based approach, we capture both of these characteristics and generate word embeddings specific to a domain.", + "Methodology ::: Dictionary-based Approach\nIn Turkish, there do not exist well-established sentiment lexicons as in English. In this approach, we made use of the TDK (Türk Dil Kurumu - “Turkish Language Institution”) dictionary to obtain word polarities." + ] + }, + "annotation_id": "1a763e25e791ca05ec7d4e88459aaae4288bafd0", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + } + ] + }, + { + "question": "How are the supervised scores of the words calculated?", + "question_id": "3355918bbdccac644afe441f085d0ffbbad565d7", + "nlp_background": "five", + "topic_background": "unfamiliar", + "paper_read": "no", + "search_query": "twitter", + "question_writer": "e8b24c3133e0bec0a6465e1f13acd3a5ed816b37", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "(+1 or -1), words of opposite polarities (e.g. “happy\" and “unhappy\") get far away from each other" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "The effect of this multiplication is exemplified in Figure FIGREF7, showing the positions of word vectors in the VSM. Those “x\" words are sentimentally negative words, those “o\" words are sentimentally positive ones. On the top coordinate plane, the words of opposite polarities are found to be close to each other, since they have common words in their dictionary definitions. Only the information concerned with the dictionary definitions are used there, discarding the polarity scores. However, when we utilise the supervised score (+1 or -1), words of opposite polarities (e.g. “happy\" and “unhappy\") get far away from each other as they are translated across coordinate regions. Positive words now appear in quadrant 1, whereas negative words appear in quadrant 3. Thus, in the VSM, words that are sentimentally similar to each other could be clustered more accurately. Besides clustering, we also employed the SVD method to perform dimensionality reduction on the unsupervised dictionary algorithm and used the newly generated matrix by combining it with other subapproaches. The number of dimensions is chosen as 200 again according to the $U$ matrix. The details are given in Section 3.4. When using and evaluating this subapproach on the English corpora, we used the SentiWordNet lexicon BIBREF13. We have achieved better results for the dictionary-based algorithm when we employed the SVD reduction method compared to the use of clustering." + ], + "highlighted_evidence": [ + "Only the information concerned with the dictionary definitions are used there, discarding the polarity scores. However, when we utilise the supervised score (+1 or -1), words of opposite polarities (e.g. “happy\" and “unhappy\") get far away from each other as they are translated across coordinate regions." + ] + }, + "annotation_id": "feecf756876f1cdf237effcdc40b9ff1d00031fa", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + } + ] + } + ], + "figures_and_tables": [ + { + "file": "5-Figure1-1.png", + "caption": "Fig. 1. The effect of using the supervised scores of words in the dictionary algorithm. It shows how sentimentally similar word vectors get closer to each other in the VSM." + }, + { + "file": "7-Figure2-1.png", + "caption": "Fig. 2. The flowchart of our system." + }, + { + "file": "9-Table1-1.png", + "caption": "Table 1. Accuracies for different feature sets fed as input into the SVM classifier in predicting the labels of reviews. The word2vec algorithm is the baseline method." + }, + { + "file": "11-Table2-1.png", + "caption": "Table 2. Most similar words to given queries according to our corpus-based approach and the baseline word2vec algorithm." + } + ] + }, + "1802.06024": { + "title": "Towards a Continuous Knowledge Learning Engine for Chatbots", + "abstract": "Although chatbots have been very popular in recent years, they still have some serious weaknesses which limit the scope of their applications. One major weakness is that they cannot learn new knowledge during the conversation process, i.e., their knowledge is fixed beforehand and cannot be expanded or updated during conversation. In this paper, we propose to build a general knowledge learning engine for chatbots to enable them to continuously and interactively learn new knowledge during conversations. As time goes by, they become more and more knowledgeable and better and better at learning and conversation. We model the task as an open-world knowledge base completion problem and propose a novel technique called lifelong interactive learning and inference (LiLi) to solve it. LiLi works by imitating how humans acquire knowledge and perform inference during an interactive conversation. Our experimental results show LiLi is highly promising.", + "full_text": [ + { + "section_name": "Introduction", + "paragraphs": [ + "Chatbots such as dialog and question-answering systems have a long history in AI and natural language processing. Early such systems were mostly built using markup languages such as AIML, handcrafted conversation generation rules, and/or information retrieval techniques BIBREF0 , BIBREF1 , BIBREF2 , BIBREF3 . Recent neural conversation models BIBREF4 , BIBREF5 , BIBREF6 are even able to perform open-ended conversations. However, since they do not use explicit knowledge bases and do not perform inference, they often suffer from generic and dull responses BIBREF5 , BIBREF7 . More recently, BIBREF8 and BIBREF9 proposed to use knowledge bases (KBs) to help generate responses for knowledge-grounded conversation. However, one major weakness of all existing chat systems is that they do not explicitly or implicitly learn new knowledge in the conversation process. This seriously limits the scope of their applications. In contrast, we humans constantly learn new knowledge in our conversations. Even if some existing systems can use very large knowledge bases either harvested from a large data source such as the Web or built manually, these KBs still miss a large number of facts (knowledge) BIBREF10 . It is thus important for a chatbot to continuously learn new knowledge in the conversation process to expand its KB and to improve its conversation ability.", + "In recent years, researchers have studied the problem of KB completion, i.e., inferring new facts (knowledge) automatically from existing facts in a KB. KB completion (KBC) is defined as a binary classification problem: Given a query triple, ( INLINEFORM0 , INLINEFORM1 , INLINEFORM2 ), we want to predict whether the source entity INLINEFORM3 and target entity INLINEFORM4 can be linked by the relation INLINEFORM5 . However, existing approaches BIBREF11 , BIBREF12 , BIBREF13 , BIBREF14 , BIBREF15 , BIBREF16 solve this problem under the closed-world assumption, i.e., INLINEFORM6 , INLINEFORM7 and INLINEFORM8 are all known to exist in the KB. This is a major weakness because it means that no new knowledge or facts may contain unknown entities or relations. Due to this limitation, KBC is clearly not sufficient for knowledge learning in conversations because in a conversation, the user can say anything, which may contain entities and relations that are not already in the KB.", + "In this paper, we remove this assumption of KBC, and allow all INLINEFORM0 , INLINEFORM1 and INLINEFORM2 to be unknown. We call the new problem open-world knowledge base completion (OKBC). OKBC generalizes KBC. Below, we show that solving OKBC naturally provides the ground for knowledge learning and inference in conversations. In essence, we formulate an abstract problem of knowledge learning and inference in conversations as a well-defined OKBC problem in the interactive setting.", + "From the perspective of knowledge learning in conversations, essentially we can extract two key types of information, true facts and queries, from the user utterances. Queries are facts whose truth values need to be determined. Note that we do not study fact or relation extraction in this paper as there is an extensive work on the topic. (1) For a true fact, we will incorporate it into the KB. Here we need to make sure that it is not already in the KB, which involves relation resolution and entity linking. After a fact is added to the KB, we may predict that some related facts involving some existing relations in the KB may also be true (not logical implications as they can be automatically inferred). For example, if the user says “Obama was born in USA,” the system may guess that (Obama, CitizenOf, USA) (meaning that Obama is a citizen of USA) could also be true based on the current KB. To verify this fact, it needs to solve a KBC problem by treating (Obama, CitizenOf, USA) as a query. This is a KBC problem because the fact (Obama, BornIn, USA) extracted from the original sentence has been added to the KB. Then Obama and USA are in the KB. If the KBC problem is solved, it learns a new fact (Obama, CitizenOf, USA) in addition to the extracted fact (Obama, BornIn, USA). (2) For a query fact, e.g., (Obama, BornIn, USA) extracted from the user question “Was Obama born in USA?” we need to solve the OKBC problem if any of “Obama, “BornIn”, or “USA\" is not already in the KB.", + "We can see that OKBC is the core of a knowledge learning engine for conversation. Thus, in this paper, we focus on solving it. We assume that other tasks such as fact/relation extraction and resolution and guessing of related facts of an extracted fact are solved by other sub-systems.", + "We solve the OKBC problem by mimicking how humans acquire knowledge and perform reasoning in an interactive conversation. Whenever we encounter an unknown concept or relation while answering a query, we perform inference using our existing knowledge. If our knowledge does not allow us to draw a conclusion, we typically ask questions to others to acquire related knowledge and use it in inference. The process typically involves an inference strategy (a sequence of actions), which interleaves a sequence of processing and interactive actions. A processing action can be the selection of related facts, deriving inference chain, etc., that advances the inference process. An interactive action can be deciding what to ask, formulating a suitable question, etc., that enable us to interact. The process helps grow the knowledge over time and the gained knowledge enables us to communicate better in the future. We call this lifelong interactive learning and inference (LiLi). Lifelong learning is reflected by the facts that the newly acquired facts are retained in the KB and used in inference for future queries, and that the accumulated knowledge in addition to the updated KB including past inference performances are leveraged to guide future interaction and learning. LiLi should have the following capabilities:", + "This setting is ideal for many NLP applications like dialog and question-answering systems that naturally provide the scope for human interaction and demand real-time inference.", + "LiLi starts with the closed-world KBC approach path-ranking (PR) BIBREF11 , BIBREF17 and extends KBC in a major way to open-world knowledge base completion (OKBC). For a relation INLINEFORM0 , PR works by enumerating paths (except single-link path INLINEFORM1 ) between entity-pairs linked by INLINEFORM2 in the KB and use them as features to train a binary classifier to predict whether a query INLINEFORM3 should be in the KB. Here, a path between two entities is a sequence of relations linking them. In our work, we adopt the latest PR method, C-PR BIBREF16 and extend it to make it work in the open-world setting. C-PR enumerates paths by performing bidirectional random walks over the KB graph while leveraging the context of the source-target entity-pair. We also adopt and extend the compositional vector space model BIBREF20 , BIBREF21 with continual learning capability for prediction.", + "Given an OKBC query ( INLINEFORM0 , INLINEFORM1 , INLINEFORM2 ) (e.g., “(Obama, CitizenOf, USA), which means whether Obama a citizen of USA), LiLi interacts with the user (if needed) by dynamically formulating questions (see the interaction example in Figure 1, which will be further explained in §3) and leverages the interactively acquired knowledge (supporting facts (SFs) in the figure) for continued inference. To do so, LiLi formulates a query-specific inference strategy and executes it. We design LiLi in a Reinforcement Learning (RL) setting that performs sub-tasks like formulating and executing strategy, training a prediction model for inference, and knowledge retention for future use. To the best of our knowledge, our work is the first to address the OKBC problem and to propose an interactive learning mechanism to solve it in a continuous or lifelong manner. We empirically verify the effectiveness of LiLi on two standard real-world KBs: Freebase and WordNet. Experimental results show that LiLi is highly effective in terms of its predictive performance and strategy formulation ability." + ] + }, + { + "section_name": "Related Work", + "paragraphs": [ + "To the best of our knowledge, we are not aware of any knowledge learning system that can learn new knowledge in the conversation process. This section thus discusses other related work.", + "Among existing KB completion approaches, BIBREF20 extended the vector space model for zero-shot KB inference. However, the model cannot handle unknown entities and can only work on fixed set of unknown relations with known embeddings. Recently, BIBREF22 proposed a method using external text corpus to perform inference on unknown entities. However, the method cannot handle unknown relations. Thus, these methods are not suitable for our open-world setting. None of the existing KB inference methods perform interactive knowledge learning like LiLi. NELL BIBREF23 continuously updates its KB using facts extracted from the Web. Our task is very different as we do not do Web fact extraction (which is also useful). We focus on user interactions in this paper. Our work is related to interactive language learning (ILL) BIBREF24 , BIBREF25 , but these are not about KB completion. The work in BIBREF26 allows a learner to ask questions in dialogue. However, this work used RL to learn about whether to ask the user or not. The “what to ask aspect\" was manually designed by modeling synthetic tasks. LiLi formulates query-specific inference strategies which embed interaction behaviors. Also, no existing dialogue systems BIBREF4 , BIBREF27 , BIBREF28 , BIBREF29 , BIBREF30 employ lifelong learning to train prediction models by using information/knowledge retained in the past.", + "Our work is related to general lifelong learning in BIBREF31 , BIBREF32 , BIBREF33 , BIBREF34 , BIBREF35 , BIBREF36 . However, they learn only one type of tasks, e.g., supervised, topic modeling or reinforcement learning (RL) tasks. None of them is suitable for our setting, which involves interleaving of RL, supervised and interactive learning. More details about lifelong learning can be found in the book BIBREF31 ." + ] + }, + { + "section_name": "Interactive Knowledge Learning (LiLi)", + "paragraphs": [ + "We design LiLi as a combination of two interconnected models: (1) a RL model that learns to formulate a query-specific inference strategy for performing the OKBC task, and (2) a lifelong prediction model to predict whether a triple should be in the KB, which is invoked by an action while executing the inference strategy and is learned for each relation as in C-PR. The framework improves its performance over time through user interaction and knowledge retention. Compared to the existing KB inference methods, LiLi overcomes the following three challenges for OKBC:", + "1. Mapping open-world to close-world. Being a closed-world method, C-PR cannot extract path features and learn a prediction model when any of INLINEFORM0 , INLINEFORM1 or INLINEFORM2 is unknown. LiLi solves this problem through interactive knowledge acquisition. If INLINEFORM3 is unknown, LiLi asks the user to provide a clue (an example of INLINEFORM4 ). And if INLINEFORM5 or INLINEFORM6 is unknown, LiLi asks the user to provide a link (relation) to connect the unknown entity with an existing entity (automatically selected) in the KB. We refer to such a query as a connecting link query (CLQ). The acquired knowledge reduces OKBC to KBC and makes the inference task feasible.", + "2. Spareseness of KB. A main issue of all PR methods like C-PR is the connectivity of the KB graph. If there is no path connecting INLINEFORM0 and INLINEFORM1 in the graph, path enumeration of C-PR gets stuck and inference becomes infeasible. In such cases, LiLi uses a template relation (“@-?-@\") as the missing link marker to connect entity-pairs and continues feature extraction. A path containing “@-?-@\" is called an incomplete path. Thus, the extracted feature set contains both complete (no missing link) and incomplete paths. Next, LiLi selects an incomplete path from the feature set and asks the user to provide a link for path completion. We refer to such a query as missing link query (MLQ).", + "3. Limitation in user knowledge. If the user is unable to respond to MLQs or CLQs, LiLi uses a guessing mechanism (discussed later) to fill the gap. This enables LiLi to continue its inference even if the user cannot answer a system question." + ] + }, + { + "section_name": "Components of LiLi", + "paragraphs": [ + "As lifelong learning needs to retain knowledge learned from past tasks and use it to help future learning BIBREF31 , LiLi uses a Knowledge Store (KS) for knowledge retention. KS has four components: (i) Knowledge Graph ( INLINEFORM0 ): INLINEFORM1 (the KB) is initialized with base KB triples (see §4) and gets updated over time with the acquired knowledge. (ii) Relation-Entity Matrix ( INLINEFORM2 ): INLINEFORM3 is a sparse matrix, with rows as relations and columns as entity-pairs and is used by the prediction model. Given a triple ( INLINEFORM4 , INLINEFORM5 , INLINEFORM6 ) INLINEFORM7 , we set INLINEFORM8 [ INLINEFORM9 , ( INLINEFORM10 , INLINEFORM11 )] = 1 indicating INLINEFORM12 occurs for pair ( INLINEFORM13 , INLINEFORM14 ). (iii) Task Experience Store ( INLINEFORM15 ): INLINEFORM16 stores the predictive performance of LiLi on past learned tasks in terms of Matthews correlation coefficient (MCC) that measures the quality of binary classification. So, for two tasks INLINEFORM17 and INLINEFORM18 (each relation is a task), if INLINEFORM19 [ INLINEFORM20 ] INLINEFORM21 INLINEFORM22 [ INLINEFORM23 ] [where INLINEFORM24 [ INLINEFORM25 ]=MCC( INLINEFORM26 )], we say C-PR has learned INLINEFORM27 well compared to INLINEFORM28 . (iv) Incomplete Feature DB ( INLINEFORM29 ): INLINEFORM30 stores the frequency of an incomplete path INLINEFORM31 in the form of a tuple ( INLINEFORM32 , INLINEFORM33 , INLINEFORM34 ) and is used in formulating MLQs. INLINEFORM35 [( INLINEFORM36 , INLINEFORM37 , INLINEFORM38 )] = INLINEFORM39 implies LiLi has extracted incomplete path INLINEFORM40 INLINEFORM41 times involving entity-pair INLINEFORM42 [( INLINEFORM43 , INLINEFORM44 )] for query relation INLINEFORM45 .", + "The RL model learns even after training whenever it encounters an unseen state (in testing) and thus, gets updated over time. KS is updated continuously over time as a result of the execution of LiLi and takes part in future learning. The prediction model uses lifelong learning (LL), where we transfer knowledge (parameter values) from the model for a past most similar task to help learn for the current task. Similar tasks are identified by factorizing INLINEFORM0 and computing a task similarity matrix INLINEFORM1 . Besides LL, LiLi uses INLINEFORM2 to identify poorly learned past tasks and acquire more clues for them to improve its skillset over time.", + "LiLi also uses a stack, called Inference Stack ( INLINEFORM0 ) to hold query and its state information for RL. LiLi always processes stack top ( INLINEFORM1 [top]). The clues from the user get stored in INLINEFORM2 on top of the query during strategy execution and processed first. Thus, the prediction model for INLINEFORM3 is learned before performing inference on query, transforming OKBC to a KBC problem. Table 1 shows the parameters of LiLi used in the following sections." + ] + }, + { + "section_name": "Working of LiLi", + "paragraphs": [ + "Given an OKBC query ( INLINEFORM0 , INLINEFORM1 , INLINEFORM2 ), we represent it as a data instance INLINEFORM3 . INLINEFORM4 consists of INLINEFORM5 (the query triple), INLINEFORM6 (interaction limit set for INLINEFORM7 ), INLINEFORM8 (experience list storing the transition history of MDP for INLINEFORM9 in RL) and INLINEFORM10 (mode of INLINEFORM11 ) denoting if INLINEFORM12 is ` INLINEFORM13 ' (training), ` INLINEFORM14 ' (validation), ` INLINEFORM15 ' (evaluation) or ` INLINEFORM16 ' (clue) instance and INLINEFORM17 (feature set). We denote INLINEFORM18 ( INLINEFORM19 ) as the set of all complete (incomplete) path features in INLINEFORM20 . Given a data instance INLINEFORM21 , LiLi starts its initialization as follows: it sets the state as INLINEFORM22 (based on INLINEFORM23 , explained later), pushes the query tuple ( INLINEFORM24 , INLINEFORM25 ) into INLINEFORM26 and feeds INLINEFORM27 [top] to the RL-model for strategy formulation from INLINEFORM28 .", + "Inference Strategy Formulation. We view solving the strategy formulation problem as learning to play an inference game, where the goal is to formulate a strategy that \"makes the inference task possible\". Considering PR methods, inference is possible, iff (1) INLINEFORM0 becomes known to its KB (by acquiring clues when INLINEFORM1 is unknown) and (2) path features are extracted between INLINEFORM2 and INLINEFORM3 (which inturn requires INLINEFORM4 and INLINEFORM5 to be known to KB). If these conditions are met at the end of an episode (when strategy formulation finishes for a given query) of the game, LiLi wins and thus, it trains the prediction model for INLINEFORM6 and uses it for inference.", + "LiLi's strategy formulation is modeled as a Markov Decision Process (MDP) with finite state ( INLINEFORM0 ) and action ( INLINEFORM1 ) spaces. A state INLINEFORM2 consists of 10 binary state variables (Table 2), each of which keeps track of results of an action INLINEFORM3 taken by LiLi and thus, records the progress in inference process made so far. INLINEFORM4 is the initial state with all state bits set as 0. If the data instance (query) is a clue [ INLINEFORM5 ], INLINEFORM6 [CLUE] is set as 1. INLINEFORM7 consists of 6 actions (Table 3). INLINEFORM8 , INLINEFORM9 , INLINEFORM10 are processing actions and INLINEFORM11 , INLINEFORM12 , INLINEFORM13 are interactive actions. Whenever INLINEFORM14 is executed, the MDP reaches the terminal state. Given an action INLINEFORM15 in state INLINEFORM16 , if INLINEFORM17 is invalid in INLINEFORM21 or the objective of INLINEFORM22 is unsatisfied (* marked the condition in INLINEFORM23 ), RL receives a negative reward (empirically set); else receives a positive reward.. We use Q-learning BIBREF38 with INLINEFORM24 -greedy strategy to learn the optimal policy for training the RL model. Note that, the inference strategy is independent of KB type and correctness of prediction. Thus, the RL-model is trained only once from scratch (reused thereafter for other KBs) and also, independently of the prediction model.", + "Sometimes the training dataset may not be enough to learn optimal policy for all INLINEFORM0 . Thus, encountering an unseen state during test can make RL-model clueless about the action. Given a state INLINEFORM1 , whenever an invalid INLINEFORM2 is chosen, LiLi remains in INLINEFORM3 . For INLINEFORM4 , LiLi remains in INLINEFORM5 untill INLINEFORM6 (see Table 1 for INLINEFORM7 ). So, if the state remains the same for ( INLINEFORM8 +1) times, it implies LiLi has encountered a fault (an unseen state). RL-model instantly switches to the training mode and randomly explores INLINEFORM9 to learn the optimal action (fault-tolerant learning). While exploring INLINEFORM10 , the model chooses INLINEFORM11 only when it has tried all other INLINEFORM12 to avoid abrupt end of episode.", + "Execution of Actions. At any given point in time, let ( INLINEFORM0 , INLINEFORM1 ) be the current INLINEFORM2 [top], INLINEFORM3 is the chosen action and the current version of KS components are INLINEFORM4 , INLINEFORM5 , INLINEFORM6 and INLINEFORM7 . Then, if INLINEFORM8 is invalid in INLINEFORM9 , LiLi only updates INLINEFORM10 [top] with ( INLINEFORM11 , INLINEFORM12 ) and returns INLINEFORM13 [top] to RL-model. In this process, LiLi adds experience ( INLINEFORM14 , INLINEFORM15 , INLINEFORM16 , INLINEFORM17 ) in INLINEFORM18 and then, replaces INLINEFORM19 [top] with ( INLINEFORM20 , INLINEFORM21 ). If INLINEFORM22 is valid in INLINEFORM23 , LiLi first sets the next state INLINEFORM24 and performs a sequence of operations INLINEFORM25 based on INLINEFORM26 (discussed below). Unless specified, in INLINEFORM27 , LiLi always monitors INLINEFORM28 and if INLINEFORM29 becomes 0, LiLi sets INLINEFORM30 . Also, whenever LiLi asks the user a query, INLINEFORM31 is decremented by 1. Once INLINEFORM32 ends, LiLi updates INLINEFORM33 [top] with ( INLINEFORM34 , INLINEFORM35 ) and returns INLINEFORM36 [top] to RL-model for choosing the next action.", + "In INLINEFORM0 , LiLi searches INLINEFORM1 , INLINEFORM2 , INLINEFORM3 in INLINEFORM4 and sets appropriate bits in INLINEFORM5 (see Table 2). If INLINEFORM6 was unknown before and is just added to INLINEFORM7 or is in the bottom INLINEFORM8 % (see Table 1 for INLINEFORM9 ) of INLINEFORM10 , LiLi randomly sets INLINEFORM14 with probability INLINEFORM15 . If INLINEFORM16 is a clue and INLINEFORM17 , LiLi updates KS with triple INLINEFORM18 , where ( INLINEFORM19 , INLINEFORM20 , INLINEFORM21 ) and ( INLINEFORM22 , INLINEFORM23 , INLINEFORM24 ) gets added to INLINEFORM25 and INLINEFORM26 , INLINEFORM27 are set as 1.", + "In INLINEFORM0 , LiLi asks the user to provide a clue (+ve instance) for INLINEFORM1 and corrupts INLINEFORM2 and INLINEFORM3 of the clue once at a time, to generate -ve instances by sampling nodes from INLINEFORM4 . These instances help in training prediction model for INLINEFORM5 while executing INLINEFORM6 .", + "In INLINEFORM0 , LiLi selects an incomplete path INLINEFORM1 from INLINEFORM2 to formulate MLQ, such that INLINEFORM3 is most frequently observed for INLINEFORM4 and INLINEFORM5 is high, given by INLINEFORM6 . Here, INLINEFORM7 denotes the contextual similarity BIBREF16 of entity-pair INLINEFORM8 . If INLINEFORM9 is high, INLINEFORM10 is more likely to possess a relation between them and so, is a good candidate for formulating MLQ. When the user does not respond to MLQ (or CLQ in INLINEFORM11 ), the guessing mechanism is used, which works as follows: Since contextual similarity of entity-pairs is highly correlated with their class labels BIBREF16 , LiLi divides the similarity range [-1, 1] into three segments, using a low ( INLINEFORM12 ) and high ( INLINEFORM13 ) similarity threshold and replaces the missing link with INLINEFORM14 in INLINEFORM15 to make it complete as follows: If INLINEFORM16 , INLINEFORM17 = “@-LooselyRelatedTo-@\"; else if INLINEFORM18 , INLINEFORM19 =“@-NotRelatedTo-@\"; Otherwise, INLINEFORM20 =“@-RelatedTo-@\".", + "In INLINEFORM0 , LiLi asks CLQs for connecting unknown entities INLINEFORM1 and/or INLINEFORM2 with INLINEFORM3 by selecting the most contextually relevant node (wrt INLINEFORM4 , INLINEFORM5 ) from INLINEFORM6 , given by link INLINEFORM7 . We adopt the contextual relevance idea in BIBREF16 which is computed using word embedding BIBREF39 ", + "In INLINEFORM0 , LiLi extracts path features INLINEFORM1 between ( INLINEFORM2 , INLINEFORM3 ) and updates INLINEFORM4 with incomplete features from INLINEFORM5 . LiLi always trains the prediction model with complete features INLINEFORM6 and once INLINEFORM7 or INLINEFORM8 , LiLi stops asking MLQs. Thus, in both INLINEFORM9 and INLINEFORM10 , LiLi always monitors INLINEFORM11 to check for the said requirements and sets INLINEFORM12 to control interactions.", + "In INLINEFORM0 , if LiLi wins the episode, it adds INLINEFORM1 in one of data buffers INLINEFORM2 based on its mode INLINEFORM3 . E.g., if INLINEFORM4 or INLINEFORM5 , INLINEFORM6 is used for training and added to INLINEFORM7 . Similarly validation buffer INLINEFORM8 and evaluation buffer INLINEFORM9 are populated. If INLINEFORM10 , LiLi invokes the prediction model for INLINEFORM11 .", + "Lifelong Relation Prediction. Given a relation INLINEFORM0 , LiLi uses INLINEFORM1 and INLINEFORM2 (see INLINEFORM3 ) to train a prediction model (say, INLINEFORM4 ) with parameters INLINEFORM5 . For a unknown INLINEFORM6 , the clue instances get stored in INLINEFORM7 and INLINEFORM8 . Thus, LiLi populates INLINEFORM9 by taking 10% (see §4) of the instances from INLINEFORM10 and starts the training. For INLINEFORM11 , LiLi uses a LSTM BIBREF40 to compose the vector representation of each feature INLINEFORM12 as INLINEFORM13 and vector representation of INLINEFORM14 as INLINEFORM15 . Next, LiLi computes the prediction value, INLINEFORM16 as sigmoid of the mean cosine similarity of all features and INLINEFORM17 , given by INLINEFORM18 ) and maximize the log-likelihood of INLINEFORM19 for training. Once INLINEFORM20 is trained, LiLi updates INLINEFORM21 [ INLINEFORM22 ] using INLINEFORM23 . We also train an inverse model for INLINEFORM24 , INLINEFORM25 by reversing the path features in INLINEFORM26 and INLINEFORM27 which help in lifelong learning (discussed below). Unlike BIBREF20 , BIBREF21 , while predicting the label for INLINEFORM28 , we compute a relation-specific prediction threshold INLINEFORM29 corresponding to INLINEFORM30 using INLINEFORM31 as: INLINEFORM32 and infer INLINEFORM33 as +ve if INLINEFORM34 and -ve otherwise. Here, INLINEFORM35 ( INLINEFORM36 ) is the mean prediction value for all +ve (-ve) examples in INLINEFORM37 .", + "Models trained on a few examples (e.g., clues acquired for unknown INLINEFORM0 ) with randomly initialized weights often perform poorly due to underfitting. Thus, we transfer knowledge (weights) from the past most similar (wrt INLINEFORM1 ) task in a lifelong learning manner BIBREF31 . LiLi uses INLINEFORM2 to find the past most similar task for INLINEFORM3 as follows: LiLi computes trancated SVD of INLINEFORM4 as INLINEFORM5 and then, the similarity matrix INLINEFORM6 . INLINEFORM7 provides the similarity between relations INLINEFORM8 and INLINEFORM9 in INLINEFORM10 . Thus, LiLi chooses a source relation INLINEFORM11 to transfer weights. Here, INLINEFORM12 is the set of all INLINEFORM13 and INLINEFORM14 for which LiLi has already learned a prediction model. Now, if INLINEFORM15 or INLINEFORM16 , LiLi randomly initializes the weights INLINEFORM17 for INLINEFORM18 and proceeds with the training. Otherwise, LiLi uses INLINEFORM19 as initial weights and fine-tunes INLINEFORM20 with a low learning rate.", + "A Running Example. Considering the example shown in Figure 1, LiLi works as follows: first, LiLi executes INLINEFORM0 and detects that the source entity “Obama\" and query relation “CitizenOf\" are unknown. Thus, LiLi executes INLINEFORM1 to acquire clue (SF1) for “CitizenOf\" and pushes the clue (+ve example) and two generated -ve examples into INLINEFORM2 . Once the clues are processed and a prediction model is trained for “CitizenOf\" by formulating separate strategies for them, LiLi becomes aware of “CitizenOf\". Now, as the clues have already been popped from INLINEFORM3 , the query becomes INLINEFORM4 and the strategy formulation process for the query resumes. Next, LiLi asks user to provide a connecting link for “Obama\" by performing INLINEFORM5 . Now, the query entities and relation being known, LiLi enumerates paths between “Obama\" and “USA\" by performing INLINEFORM6 . Let an extracted path be “ INLINEFORM7 \" with missing link between ( INLINEFORM8 , INLINEFORM9 ). LiLi asks the user to fill the link by performing INLINEFORM10 and then, extracts the complete feature “ INLINEFORM11 \". The feature set is then fed to the prediction model and inference is made as a result of INLINEFORM12 . Thus, the formulated inference strategy is: “ INLINEFORM13 \"." + ] + }, + { + "section_name": "Experiments", + "paragraphs": [ + "We now evaluate LiLi in terms of its predictive performance and strategy formulation abilities.", + "Data: We use two standard datasets (see Table 4): (1) Freebase FB15k, and (2) WordNet INLINEFORM0 . Using each dataset, we build a fairly large graph and use it as the original KB ( INLINEFORM1 ) for evaluation. We also augment INLINEFORM2 with inverse triples ( INLINEFORM3 , INLINEFORM4 , INLINEFORM5 ) for each ( INLINEFORM6 , INLINEFORM7 , INLINEFORM8 ) following existing KBC methods.", + "Parameter Settings. Unless specified, the empirically set parameters (see Table 1) of LiLi are: INLINEFORM0 , INLINEFORM1 , INLINEFORM2 , INLINEFORM3 , INLINEFORM4 , INLINEFORM5 , INLINEFORM6 , INLINEFORM7 , INLINEFORM8 , INLINEFORM9 , INLINEFORM10 . For training RL-model with INLINEFORM11 -greedy strategy, we use INLINEFORM12 , INLINEFORM13 , pre-training steps=50000. We used Keras deep learning library to implement and train the prediction model. We set batch-size as 128, max. training epoch as 150, dropout as 0.2, hidden units and embedding size as 300 and learning rate as 5e-3 which is reduced gradually on plateau with factor 0.5 and patience 5. Adam optimizer and early stopping were used in training. We also shuffle INLINEFORM14 in each epoch and adjust class weights inversely proportional to class frequencies in INLINEFORM15 .", + "Labeled Dataset Generation and Simulated User Creation. We create a simulated user for each KB to evaluate LiLi. We create the labeled datasets, the simulated user’s knowledge base ( INLINEFORM0 ), and the base KB ( INLINEFORM1 ) from INLINEFORM2 . INLINEFORM3 used as the initial KB graph ( INLINEFORM4 ) of LiLi.", + "We followed BIBREF16 for labeled dataset generation. For Freebase, we found 86 relations with INLINEFORM0 triples and randomly selected 50 from various domains. We randomly shuffle the list of 50 relations, select 25% of them as unknown relations and consider the rest (75%) as known relations. For each known relation INLINEFORM1 , we randomly shuffle the list of distinct triples for INLINEFORM2 , choose 1000 triples and split them into 60% training, 10% validation and 20% test. Rest 10% along with the leftover (not included in the list of 1000) triples are added to INLINEFORM3 . For each unknown relation INLINEFORM4 , we remove all triples of INLINEFORM5 from INLINEFORM6 and add them to INLINEFORM7 . In this process, we also randomly choose 20% triples as test instances for unknown INLINEFORM8 which are excluded from INLINEFORM9 . Note that, now INLINEFORM10 has at least 10% of chosen triples for each INLINEFORM11 (known and unknown) and so, user is always able to provide clues for both cases. For each labeled dataset, we randomly choose 10% of the entities present in dataset triples, remove triples involving those entities from INLINEFORM12 and add to INLINEFORM13 . At this point, INLINEFORM14 gets reduced to INLINEFORM15 and is used as INLINEFORM16 for LiLi. The dataset stats in Table 4 shows that the base KB (60% triples of INLINEFORM17 ) is highly sparse (compared to original KB) which makes the inference task much harder. WordNet dataset being small, we select all 18 relations for evaluation and create labeled dataset, INLINEFORM18 and INLINEFORM19 following Freebase. Although the user may provide clues 100% of the time, it often cannot respond to MLQs and CLQs (due to lack of required triples/facts). Thus, we further enrich INLINEFORM20 with external KB triples.", + "Given a relation INLINEFORM0 and an observed triple ( INLINEFORM1 , INLINEFORM2 , INLINEFORM3 ) in training or testing, the pair ( INLINEFORM4 , INLINEFORM5 ) is regarded as a +ve instance for INLINEFORM6 . Following BIBREF18 , for each +ve instance ( INLINEFORM7 , INLINEFORM8 ), we generate two negative ones, one by randomly corrupting the source INLINEFORM9 , and the other by corrupting the target INLINEFORM10 . Note that, the test triples are not in INLINEFORM11 or INLINEFORM12 and none of the -ve instances overlap with the +ve ones.", + "Baselines. As none of the existing KBC methods can solve the OKBC problem, we choose various versions of LiLi as baselines.", + "Single: Version of LiLi where we train a single prediction model INLINEFORM0 for all test relations.", + "Sep: We do not transfer (past learned) weights for initializing INLINEFORM0 , i.e., we disable LL.", + "F-th): Here, we use a fixed prediction threshold 0.5 instead of relation-specific threshold INLINEFORM0 .", + "BG: The missing or connecting links (when the user does not respond) are filled with “@-RelatedTo-@\" blindly, no guessing mechanism.", + "w/o PTS: LiLi does not ask for additional clues via past task selection for skillset improvement.", + "Evaluation Metrics. To evaluate the strategy formulation ability, we introduce a measure called Coverage( INLINEFORM0 ), defined as the fraction of total query data instances, for which LiLi has successfully formulated strategies that lead to winning. If LiLi wins on all episodes for a given dataset, INLINEFORM1 is 1.0. To evaluate the predictive performance, we use Avg. MCC and avg. +ve F1 score." + ] + }, + { + "section_name": "Results and Analysis", + "paragraphs": [ + "Evaluation-I: Strategy Formulation Ability. Table 5 shows the list of inference strategies formulated by LiLi for various INLINEFORM0 and INLINEFORM1 , which control the strategy formulation of LiLi. When INLINEFORM2 , LiLi cannot interact with user and works like a closed-world method. Thus, INLINEFORM3 drops significantly (0.47). When INLINEFORM4 , i.e. with only one interaction per query, LiLi acquires knowledge well for instances where either of the entities or relation is unknown. However, as one unknown entity may appear in multiple test triples, once the entity becomes known, LiLi doesn’t need to ask for it again and can perform inference on future triples causing significant increase in INLINEFORM5 (0.97). When INLINEFORM6 , LiLi is able to perform inference on all instances and INLINEFORM7 becomes 1. For INLINEFORM8 , LiLi uses INLINEFORM9 only once (as only one MLQ satisfies INLINEFORM10 ) compared to INLINEFORM11 . In summary, LiLi’s RL-model can effectively formulate query-specific inference strategies (based on specified parameter values). Evaluation-II: Predictive Performance. Table 6 shows the comparative performance of LiLi with baselines. To judge the overall improvements, we performed paired t-test considering +ve F1 scores on each relation as paired data. Considering both KBs and all relation types, LiLi outperforms Sep with INLINEFORM12 . If we set INLINEFORM13 (training with very few clues), LiLi outperforms Sep with INLINEFORM14 on Freebase considering MCC. Thus, the lifelong learning mechanism is effective in transferring helpful knowledge. Single model performs better than Sep for unknown relations due to the sharing of knowledge (weights) across tasks. However, for known relations, performance drops because, as a new relation arrives to the system, old weights get corrupted and catastrophic forgetting occurs. For unknown relations, as the relations are evaluated just after training, there is no chance for catastrophic forgetting. The performance improvement ( INLINEFORM15 ) of LiLi over F-th on Freebase signifies that the relation-specific threshold INLINEFORM16 works better than fixed threshold 0.5 because, if all prediction values for test instances lie above (or below) 0.5, F-th predicts all instances as +ve (-ve) which degrades its performance. Due to the utilization of contextual similarity (highly correlated with class labels) of entity-pairs, LiLi’s guessing mechanism works better ( INLINEFORM17 ) than blind guessing (BG). The past task selection mechanism of LiLi also improves its performance over w/o PTS, as it acquires more clues during testing for poorly performed tasks (evaluated on validation set). For Freebase, due to a large number of past tasks [9 (25% of 38)], the performance difference is more significant ( INLINEFORM18 ). For WordNet, the number is relatively small [3 (25% of 14)] and hence, the difference is not significant.", + "Evaluation-III: User Interaction vs. Performance. Table 7 shows the results of LiLi by varying clue acquisition rate ( INLINEFORM0 ). We use Freebase for tuning INLINEFORM1 due to its higher number of unknown test relations compared to WordNet. LiLi’s performance improves significantly as it acquires more clues from the user. The results on INLINEFORM2 outperforms ( INLINEFORM3 ) that on INLINEFORM4 . Table 8 shows the results of LiLi on user responses to MLQ’s and CLQ’s. Answering MLQ’s and CLQ’s is very hard for simulated users (unlike crowd-sourcing) as often INLINEFORM5 lacks the required triple. Thus, we attempt to analyze how the performance is effected if the user does not respond at all. The results show a clear trend in overall performance improvement when the user responds. However, the improvement is not significant as the simulated user’s query satisfaction rate (1% MLQs and 10% CLQs) is very small. But, the analysis shows the effectiveness of LiLi’s guessing mechanism and continual learning ability that help in achieving avg. +ve F1 of 0.57 and 0.62 on FB and WN respectively with minimal participation of the user." + ] + }, + { + "section_name": "Conclusion", + "paragraphs": [ + " In this paper, we are interested in building a generic engine for continuous knowledge learning in human-machine conversations. We first showed that the problem underlying the engine can be formulated as an open-world knowledge base completion (OKBC) problem. We then proposed an lifelong interactive learning and inference (LiLi) approach to solving the OKBC problem. OKBC is a generalization of KBC. LiLi solves the OKBC problem by first formulating a query-specific inference strategy using RL and then executing it to solve the problem by interacting with the user in a lifelong learning manner. Experimental results showed the effectiveness of LiLi in terms of both predictive quality and strategy formulation ability. We believe that a system with the LiLi approach can serve as a knowledge learning engine for conversations. Our future work will improve LiLi to make more accurate." + ] + }, + { + "section_name": "Acknowledgments", + "paragraphs": [ + "This work was supported in part by National Science Foundation (NSF) under grant no. IIS-1407927 and IIS-1650900, and a gift from Huawei Technologies Co Ltd." + ] + } + ], + "qas": [ + { + "question": "Do they report results only on English data?", + "question_id": "cb196725edc9cdb2c54b72364f3bbf7c76471490", + "nlp_background": "five", + "topic_background": "", + "paper_read": "", + "search_query": "", + "question_writer": "e8b24c3133e0bec0a6465e1f13acd3a5ed816b37", + "answers": [ + { + "answer": { + "unanswerable": true, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "", + "evidence": [], + "highlighted_evidence": [] + }, + "annotation_id": "8c5faa91a3736810aa4747688b8a5eea5cf19c21", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + }, + { + "answer": { + "unanswerable": true, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "", + "evidence": [], + "highlighted_evidence": [] + }, + "annotation_id": "cdca429774324ce7ceaecebbef456c09f81c9b6f", + "worker_id": "a0b403873302db7cada39008f04d01155ef68f4f" + } + ] + }, + { + "question": "How much better than the baseline is LiLi?", + "question_id": "286078813136943dfafb5155ee15d2429e7601d9", + "nlp_background": "five", + "topic_background": "", + "paper_read": "", + "search_query": "", + "question_writer": "e8b24c3133e0bec0a6465e1f13acd3a5ed816b37", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "In case of Freebase knowledge base, LiLi model had better F1 score than the single model by 0.20 , 0.01, 0.159 for kwn, unk, and all test Rel type. The values for WordNet are 0.25, 0.1, 0.2. \n", + "evidence": [ + "Baselines. As none of the existing KBC methods can solve the OKBC problem, we choose various versions of LiLi as baselines.", + "Single: Version of LiLi where we train a single prediction model INLINEFORM0 for all test relations.", + "Sep: We do not transfer (past learned) weights for initializing INLINEFORM0 , i.e., we disable LL.", + "F-th): Here, we use a fixed prediction threshold 0.5 instead of relation-specific threshold INLINEFORM0 .", + "BG: The missing or connecting links (when the user does not respond) are filled with “@-RelatedTo-@\" blindly, no guessing mechanism.", + "w/o PTS: LiLi does not ask for additional clues via past task selection for skillset improvement.", + "Evaluation-I: Strategy Formulation Ability. Table 5 shows the list of inference strategies formulated by LiLi for various INLINEFORM0 and INLINEFORM1 , which control the strategy formulation of LiLi. When INLINEFORM2 , LiLi cannot interact with user and works like a closed-world method. Thus, INLINEFORM3 drops significantly (0.47). When INLINEFORM4 , i.e. with only one interaction per query, LiLi acquires knowledge well for instances where either of the entities or relation is unknown. However, as one unknown entity may appear in multiple test triples, once the entity becomes known, LiLi doesn’t need to ask for it again and can perform inference on future triples causing significant increase in INLINEFORM5 (0.97). When INLINEFORM6 , LiLi is able to perform inference on all instances and INLINEFORM7 becomes 1. For INLINEFORM8 , LiLi uses INLINEFORM9 only once (as only one MLQ satisfies INLINEFORM10 ) compared to INLINEFORM11 . In summary, LiLi’s RL-model can effectively formulate query-specific inference strategies (based on specified parameter values). Evaluation-II: Predictive Performance. Table 6 shows the comparative performance of LiLi with baselines. To judge the overall improvements, we performed paired t-test considering +ve F1 scores on each relation as paired data. Considering both KBs and all relation types, LiLi outperforms Sep with INLINEFORM12 . If we set INLINEFORM13 (training with very few clues), LiLi outperforms Sep with INLINEFORM14 on Freebase considering MCC. Thus, the lifelong learning mechanism is effective in transferring helpful knowledge. Single model performs better than Sep for unknown relations due to the sharing of knowledge (weights) across tasks. However, for known relations, performance drops because, as a new relation arrives to the system, old weights get corrupted and catastrophic forgetting occurs. For unknown relations, as the relations are evaluated just after training, there is no chance for catastrophic forgetting. The performance improvement ( INLINEFORM15 ) of LiLi over F-th on Freebase signifies that the relation-specific threshold INLINEFORM16 works better than fixed threshold 0.5 because, if all prediction values for test instances lie above (or below) 0.5, F-th predicts all instances as +ve (-ve) which degrades its performance. Due to the utilization of contextual similarity (highly correlated with class labels) of entity-pairs, LiLi’s guessing mechanism works better ( INLINEFORM17 ) than blind guessing (BG). The past task selection mechanism of LiLi also improves its performance over w/o PTS, as it acquires more clues during testing for poorly performed tasks (evaluated on validation set). For Freebase, due to a large number of past tasks [9 (25% of 38)], the performance difference is more significant ( INLINEFORM18 ). For WordNet, the number is relatively small [3 (25% of 14)] and hence, the difference is not significant.", + "FLOAT SELECTED: Table 6: Comparison of predictive performance of various versions of LiLi [kwn = known, unk = unknown, all = overall]." + ], + "highlighted_evidence": [ + "Baselines. As none of the existing KBC methods can solve the OKBC problem, we choose various versions of LiLi as baselines.\n\nSingle: Version of LiLi where we train a single prediction model INLINEFORM0 for all test relations.\n\nSep: We do not transfer (past learned) weights for initializing INLINEFORM0 , i.e., we disable LL.\n\nF-th): Here, we use a fixed prediction threshold 0.5 instead of relation-specific threshold INLINEFORM0 .\n\nBG: The missing or connecting links (when the user does not respond) are filled with “@-RelatedTo-@\" blindly, no guessing mechanism.\n\nw/o PTS: LiLi does not ask for additional clues via past task selection for skillset improvement.", + "Table 6 shows the comparative performance of LiLi with baselines. To judge the overall improvements, we performed paired t-test considering +ve F1 scores on each relation as paired data. Considering both KBs and all relation types, LiLi outperforms Sep with INLINEFORM12 . If we set INLINEFORM13 (training with very few clues), LiLi outperforms Sep with INLINEFORM14 on Freebase considering MCC. Thus, the lifelong learning mechanism is effective in transferring helpful knowledge. ", + "FLOAT SELECTED: Table 6: Comparison of predictive performance of various versions of LiLi [kwn = known, unk = unknown, all = overall]." + ] + }, + "annotation_id": "8358504a434f7414031f550d6b2d7eeaf258096c", + "worker_id": "a0b403873302db7cada39008f04d01155ef68f4f" + }, + { + "answer": { + "unanswerable": true, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "", + "evidence": [], + "highlighted_evidence": [] + }, + "annotation_id": "eeba7d1dafb368f928402450a7d34b5462e7e526", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + } + ] + }, + { + "question": "What baseline is used in the experiments?", + "question_id": "8f16dc7d7be0d284069841e456ebb2c69575b32b", + "nlp_background": "five", + "topic_background": "", + "paper_read": "", + "search_query": "", + "question_writer": "e8b24c3133e0bec0a6465e1f13acd3a5ed816b37", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "versions of LiLi" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "Baselines. As none of the existing KBC methods can solve the OKBC problem, we choose various versions of LiLi as baselines.", + "Single: Version of LiLi where we train a single prediction model INLINEFORM0 for all test relations.", + "Sep: We do not transfer (past learned) weights for initializing INLINEFORM0 , i.e., we disable LL.", + "F-th): Here, we use a fixed prediction threshold 0.5 instead of relation-specific threshold INLINEFORM0 .", + "BG: The missing or connecting links (when the user does not respond) are filled with “@-RelatedTo-@\" blindly, no guessing mechanism.", + "w/o PTS: LiLi does not ask for additional clues via past task selection for skillset improvement." + ], + "highlighted_evidence": [ + "Baselines. As none of the existing KBC methods can solve the OKBC problem, we choose various versions of LiLi as baselines.\n\nSingle: Version of LiLi where we train a single prediction model INLINEFORM0 for all test relations.\n\nSep: We do not transfer (past learned) weights for initializing INLINEFORM0 , i.e., we disable LL.\n\nF-th): Here, we use a fixed prediction threshold 0.5 instead of relation-specific threshold INLINEFORM0 .\n\nBG: The missing or connecting links (when the user does not respond) are filled with “@-RelatedTo-@\" blindly, no guessing mechanism.\n\nw/o PTS: LiLi does not ask for additional clues via past task selection for skillset improvement." + ] + }, + "annotation_id": "3857990188d472168e51eba265a29b997d109b72", + "worker_id": "a0b403873302db7cada39008f04d01155ef68f4f" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "various versions of LiLi as baselines", + "Single", + "Sep", + "F-th", + "BG", + "w/o PTS" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "Baselines. As none of the existing KBC methods can solve the OKBC problem, we choose various versions of LiLi as baselines.", + "Single: Version of LiLi where we train a single prediction model INLINEFORM0 for all test relations.", + "Sep: We do not transfer (past learned) weights for initializing INLINEFORM0 , i.e., we disable LL.", + "F-th): Here, we use a fixed prediction threshold 0.5 instead of relation-specific threshold INLINEFORM0 .", + "BG: The missing or connecting links (when the user does not respond) are filled with “@-RelatedTo-@\" blindly, no guessing mechanism.", + "w/o PTS: LiLi does not ask for additional clues via past task selection for skillset improvement." + ], + "highlighted_evidence": [ + "Baselines. As none of the existing KBC methods can solve the OKBC problem, we choose various versions of LiLi as baselines.\n\nSingle: Version of LiLi where we train a single prediction model INLINEFORM0 for all test relations.\n\nSep: We do not transfer (past learned) weights for initializing INLINEFORM0 , i.e., we disable LL.\n\nF-th): Here, we use a fixed prediction threshold 0.5 instead of relation-specific threshold INLINEFORM0 .\n\nBG: The missing or connecting links (when the user does not respond) are filled with “@-RelatedTo-@\" blindly, no guessing mechanism.\n\nw/o PTS: LiLi does not ask for additional clues via past task selection for skillset improvement." + ] + }, + "annotation_id": "8bb504ad711785740c881d1987f2ee6d1e9ac7d5", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + } + ] + }, + { + "question": "In what way does LiLi imitate how humans acquire knowledge and perform inference during an interactive conversation?", + "question_id": "a7d020120a45c39bee624f65443e09b895c10533", + "nlp_background": "five", + "topic_background": "", + "paper_read": "", + "search_query": "", + "question_writer": "e8b24c3133e0bec0a6465e1f13acd3a5ed816b37", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "newly acquired facts are retained in the KB and used in inference for future queries, and that the accumulated knowledge in addition to the updated KB including past inference performances are leveraged to guide future interaction and learning" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "We solve the OKBC problem by mimicking how humans acquire knowledge and perform reasoning in an interactive conversation. Whenever we encounter an unknown concept or relation while answering a query, we perform inference using our existing knowledge. If our knowledge does not allow us to draw a conclusion, we typically ask questions to others to acquire related knowledge and use it in inference. The process typically involves an inference strategy (a sequence of actions), which interleaves a sequence of processing and interactive actions. A processing action can be the selection of related facts, deriving inference chain, etc., that advances the inference process. An interactive action can be deciding what to ask, formulating a suitable question, etc., that enable us to interact. The process helps grow the knowledge over time and the gained knowledge enables us to communicate better in the future. We call this lifelong interactive learning and inference (LiLi). Lifelong learning is reflected by the facts that the newly acquired facts are retained in the KB and used in inference for future queries, and that the accumulated knowledge in addition to the updated KB including past inference performances are leveraged to guide future interaction and learning. LiLi should have the following capabilities:" + ], + "highlighted_evidence": [ + "We call this lifelong interactive learning and inference (LiLi). Lifelong learning is reflected by the facts that the newly acquired facts are retained in the KB and used in inference for future queries, and that the accumulated knowledge in addition to the updated KB including past inference performances are leveraged to guide future interaction and learning." + ] + }, + "annotation_id": "7c018db97e98972e0d0f26697a7830456982dff5", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "Whenever we encounter an unknown concept or relation while answering a query, we perform inference using our existing knowledge. If our knowledge does not allow us to draw a conclusion, we typically ask questions to others to acquire related knowledge and use it in inference. " + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "We solve the OKBC problem by mimicking how humans acquire knowledge and perform reasoning in an interactive conversation. Whenever we encounter an unknown concept or relation while answering a query, we perform inference using our existing knowledge. If our knowledge does not allow us to draw a conclusion, we typically ask questions to others to acquire related knowledge and use it in inference. The process typically involves an inference strategy (a sequence of actions), which interleaves a sequence of processing and interactive actions. A processing action can be the selection of related facts, deriving inference chain, etc., that advances the inference process. An interactive action can be deciding what to ask, formulating a suitable question, etc., that enable us to interact. The process helps grow the knowledge over time and the gained knowledge enables us to communicate better in the future. We call this lifelong interactive learning and inference (LiLi). Lifelong learning is reflected by the facts that the newly acquired facts are retained in the KB and used in inference for future queries, and that the accumulated knowledge in addition to the updated KB including past inference performances are leveraged to guide future interaction and learning. LiLi should have the following capabilities:" + ], + "highlighted_evidence": [ + "We solve the OKBC problem by mimicking how humans acquire knowledge and perform reasoning in an interactive conversation. Whenever we encounter an unknown concept or relation while answering a query, we perform inference using our existing knowledge. If our knowledge does not allow us to draw a conclusion, we typically ask questions to others to acquire related knowledge and use it in inference. The process typically involves an inference strategy (a sequence of actions), which interleaves a sequence of processing and interactive actions. " + ] + }, + "annotation_id": "a68d8bd0218ad29e5b886400df91deee1d28e9a4", + "worker_id": "a0b403873302db7cada39008f04d01155ef68f4f" + } + ] + }, + { + "question": "What metrics are used to establish that this makes chatbots more knowledgeable and better at learning and conversation? ", + "question_id": "585626d18a20d304ae7df228c2128da542d248ff", + "nlp_background": "five", + "topic_background": "", + "paper_read": "", + "search_query": "", + "question_writer": "e8b24c3133e0bec0a6465e1f13acd3a5ed816b37", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "Coverage", + "Avg. MCC and avg. +ve F1 score" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "Evaluation Metrics. To evaluate the strategy formulation ability, we introduce a measure called Coverage( INLINEFORM0 ), defined as the fraction of total query data instances, for which LiLi has successfully formulated strategies that lead to winning. If LiLi wins on all episodes for a given dataset, INLINEFORM1 is 1.0. To evaluate the predictive performance, we use Avg. MCC and avg. +ve F1 score." + ], + "highlighted_evidence": [ + "Evaluation Metrics. To evaluate the strategy formulation ability, we introduce a measure called Coverage( INLINEFORM0 ), defined as the fraction of total query data instances, for which LiLi has successfully formulated strategies that lead to winning. If LiLi wins on all episodes for a given dataset, INLINEFORM1 is 1.0. To evaluate the predictive performance, we use Avg. MCC and avg. +ve F1 score." + ] + }, + "annotation_id": "48461765d09f6eb41b036421fb6b7d261f3cb49e", + "worker_id": "a0b403873302db7cada39008f04d01155ef68f4f" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "strategy formulation ability, we introduce a measure called Coverage( INLINEFORM0 )", + "To evaluate the predictive performance, we use Avg. MCC and avg. +ve F1 score" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "Evaluation Metrics. To evaluate the strategy formulation ability, we introduce a measure called Coverage( INLINEFORM0 ), defined as the fraction of total query data instances, for which LiLi has successfully formulated strategies that lead to winning. If LiLi wins on all episodes for a given dataset, INLINEFORM1 is 1.0. To evaluate the predictive performance, we use Avg. MCC and avg. +ve F1 score." + ], + "highlighted_evidence": [ + "To evaluate the strategy formulation ability, we introduce a measure called Coverage( INLINEFORM0 ), defined as the fraction of total query data instances, for which LiLi has successfully formulated strategies that lead to winning. If LiLi wins on all episodes for a given dataset, INLINEFORM1 is 1.0. To evaluate the predictive performance, we use Avg. MCC and avg. +ve F1 score." + ] + }, + "annotation_id": "6618ac2f2bde471a9ccd963366fb4bed24159aa4", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + } + ] + }, + { + "question": "What are the components of the general knowledge learning engine?", + "question_id": "bfc2dc913e7b78f3bd45e5449d71383d0aa4a890", + "nlp_background": "five", + "topic_background": "", + "paper_read": "", + "search_query": "", + "question_writer": "e8b24c3133e0bec0a6465e1f13acd3a5ed816b37", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "Answer with content missing: (list)\nLiLi should have the following capabilities:\n1. to formulate an inference strategy for a given query that embeds processing and interactive actions.\n2. to learn interaction behaviors (deciding what to ask and when to ask the user).\n3. to leverage the acquired knowledge in the current and future inference process.\n4. to perform 1, 2 and 3 in a lifelong manner for continuous knowledge learning.", + "evidence": [ + "We solve the OKBC problem by mimicking how humans acquire knowledge and perform reasoning in an interactive conversation. Whenever we encounter an unknown concept or relation while answering a query, we perform inference using our existing knowledge. If our knowledge does not allow us to draw a conclusion, we typically ask questions to others to acquire related knowledge and use it in inference. The process typically involves an inference strategy (a sequence of actions), which interleaves a sequence of processing and interactive actions. A processing action can be the selection of related facts, deriving inference chain, etc., that advances the inference process. An interactive action can be deciding what to ask, formulating a suitable question, etc., that enable us to interact. The process helps grow the knowledge over time and the gained knowledge enables us to communicate better in the future. We call this lifelong interactive learning and inference (LiLi). Lifelong learning is reflected by the facts that the newly acquired facts are retained in the KB and used in inference for future queries, and that the accumulated knowledge in addition to the updated KB including past inference performances are leveraged to guide future interaction and learning. LiLi should have the following capabilities:" + ], + "highlighted_evidence": [ + "LiLi should have the following capabilities:" + ] + }, + "annotation_id": "a3cb3b3427bc37ef26476cbfac39951c268270e1", + "worker_id": "258ee4069f740c400c0049a2580945a1cc7f044c" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "Knowledge Store (KS) ", + "Knowledge Graph ( INLINEFORM0 )", + " Relation-Entity Matrix ( INLINEFORM2 )", + "Task Experience Store ( INLINEFORM15 )", + "Incomplete Feature DB ( INLINEFORM29 )" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "As lifelong learning needs to retain knowledge learned from past tasks and use it to help future learning BIBREF31 , LiLi uses a Knowledge Store (KS) for knowledge retention. KS has four components: (i) Knowledge Graph ( INLINEFORM0 ): INLINEFORM1 (the KB) is initialized with base KB triples (see §4) and gets updated over time with the acquired knowledge. (ii) Relation-Entity Matrix ( INLINEFORM2 ): INLINEFORM3 is a sparse matrix, with rows as relations and columns as entity-pairs and is used by the prediction model. Given a triple ( INLINEFORM4 , INLINEFORM5 , INLINEFORM6 ) INLINEFORM7 , we set INLINEFORM8 [ INLINEFORM9 , ( INLINEFORM10 , INLINEFORM11 )] = 1 indicating INLINEFORM12 occurs for pair ( INLINEFORM13 , INLINEFORM14 ). (iii) Task Experience Store ( INLINEFORM15 ): INLINEFORM16 stores the predictive performance of LiLi on past learned tasks in terms of Matthews correlation coefficient (MCC) that measures the quality of binary classification. So, for two tasks INLINEFORM17 and INLINEFORM18 (each relation is a task), if INLINEFORM19 [ INLINEFORM20 ] INLINEFORM21 INLINEFORM22 [ INLINEFORM23 ] [where INLINEFORM24 [ INLINEFORM25 ]=MCC( INLINEFORM26 )], we say C-PR has learned INLINEFORM27 well compared to INLINEFORM28 . (iv) Incomplete Feature DB ( INLINEFORM29 ): INLINEFORM30 stores the frequency of an incomplete path INLINEFORM31 in the form of a tuple ( INLINEFORM32 , INLINEFORM33 , INLINEFORM34 ) and is used in formulating MLQs. INLINEFORM35 [( INLINEFORM36 , INLINEFORM37 , INLINEFORM38 )] = INLINEFORM39 implies LiLi has extracted incomplete path INLINEFORM40 INLINEFORM41 times involving entity-pair INLINEFORM42 [( INLINEFORM43 , INLINEFORM44 )] for query relation INLINEFORM45 .", + "The RL model learns even after training whenever it encounters an unseen state (in testing) and thus, gets updated over time. KS is updated continuously over time as a result of the execution of LiLi and takes part in future learning. The prediction model uses lifelong learning (LL), where we transfer knowledge (parameter values) from the model for a past most similar task to help learn for the current task. Similar tasks are identified by factorizing INLINEFORM0 and computing a task similarity matrix INLINEFORM1 . Besides LL, LiLi uses INLINEFORM2 to identify poorly learned past tasks and acquire more clues for them to improve its skillset over time.", + "LiLi also uses a stack, called Inference Stack ( INLINEFORM0 ) to hold query and its state information for RL. LiLi always processes stack top ( INLINEFORM1 [top]). The clues from the user get stored in INLINEFORM2 on top of the query during strategy execution and processed first. Thus, the prediction model for INLINEFORM3 is learned before performing inference on query, transforming OKBC to a KBC problem. Table 1 shows the parameters of LiLi used in the following sections." + ], + "highlighted_evidence": [ + "As lifelong learning needs to retain knowledge learned from past tasks and use it to help future learning BIBREF31 , LiLi uses a Knowledge Store (KS) for knowledge retention. KS has four components: (i) Knowledge Graph ( INLINEFORM0 ): INLINEFORM1 (the KB) is initialized with base KB triples (see §4) and gets updated over time with the acquired knowledge. (ii) Relation-Entity Matrix ( INLINEFORM2 ): INLINEFORM3 is a sparse matrix, with rows as relations and columns as entity-pairs and is used by the prediction model. Given a triple ( INLINEFORM4 , INLINEFORM5 , INLINEFORM6 ) INLINEFORM7 , we set INLINEFORM8 [ INLINEFORM9 , ( INLINEFORM10 , INLINEFORM11 )] = 1 indicating INLINEFORM12 occurs for pair ( INLINEFORM13 , INLINEFORM14 ). (iii) Task Experience Store ( INLINEFORM15 ): INLINEFORM16 stores the predictive performance of LiLi on past learned tasks in terms of Matthews correlation coefficient (MCC) that measures the quality of binary classification. So, for two tasks INLINEFORM17 and INLINEFORM18 (each relation is a task), if INLINEFORM19 [ INLINEFORM20 ] INLINEFORM21 INLINEFORM22 [ INLINEFORM23 ] [where INLINEFORM24 [ INLINEFORM25 ]=MCC( INLINEFORM26 )], we say C-PR has learned INLINEFORM27 well compared to INLINEFORM28 . (iv) Incomplete Feature DB ( INLINEFORM29 ): INLINEFORM30 stores the frequency of an incomplete path INLINEFORM31 in the form of a tuple ( INLINEFORM32 , INLINEFORM33 , INLINEFORM34 ) and is used in formulating MLQs. INLINEFORM35 [( INLINEFORM36 , INLINEFORM37 , INLINEFORM38 )] = INLINEFORM39 implies LiLi has extracted incomplete path INLINEFORM40 INLINEFORM41 times involving entity-pair INLINEFORM42 [( INLINEFORM43 , INLINEFORM44 )] for query relation INLINEFORM45 .\n\nThe RL model learns even after training whenever it encounters an unseen state (in testing) and thus, gets updated over time. KS is updated continuously over time as a result of the execution of LiLi and takes part in future learning. The prediction model uses lifelong learning (LL), where we transfer knowledge (parameter values) from the model for a past most similar task to help learn for the current task. Similar tasks are identified by factorizing INLINEFORM0 and computing a task similarity matrix INLINEFORM1 . Besides LL, LiLi uses INLINEFORM2 to identify poorly learned past tasks and acquire more clues for them to improve its skillset over time.\n\nLiLi also uses a stack, called Inference Stack ( INLINEFORM0 ) to hold query and its state information for RL. LiLi always processes stack top ( INLINEFORM1 [top]). The clues from the user get stored in INLINEFORM2 on top of the query during strategy execution and processed first. Thus, the prediction model for INLINEFORM3 is learned before performing inference on query, transforming OKBC to a KBC problem. Table 1 shows the parameters of LiLi used in the following sections." + ] + }, + "annotation_id": "e323ec718eef3f5a8a4b336b2c5948d0768cd193", + "worker_id": "a0b403873302db7cada39008f04d01155ef68f4f" + } + ] + } + ], + "figures_and_tables": [ + { + "file": "3-Figure1-1.png", + "caption": "Figure 1: An example of interactive inference and learning. Note that LiLi only works with triples. Each triple above is assumed to be extracted from the sentence after it." + }, + { + "file": "5-Table2-1.png", + "caption": "Table 2: State bits and their meanings." + }, + { + "file": "5-Table3-1.png", + "caption": "Table 3: Actions and their descriptions." + }, + { + "file": "7-Table4-1.png", + "caption": "Table 4: Dataset statistics [kwn = known, unk = unknown]" + }, + { + "file": "8-Table5-1.png", + "caption": "Table 5: Inference strategies formulated by LiLi (ordered by frequency)." + }, + { + "file": "8-Table6-1.png", + "caption": "Table 6: Comparison of predictive performance of various versions of LiLi [kwn = known, unk = unknown, all = overall]." + }, + { + "file": "9-Table8-1.png", + "caption": "Table 8: Performance of LiLi on user’s responses." + } + ] + }, + "1905.11268": { + "title": "Combating Adversarial Misspellings with Robust Word Recognition", + "abstract": "To combat adversarial spelling mistakes, we propose placing a word recognition model in front of the downstream classifier. Our word recognition models build upon the RNN semicharacter architecture, introducing several new backoff strategies for handling rare and unseen words. Trained to recognize words corrupted by random adds, drops, swaps, and keyboard mistakes, our method achieves 32% relative (and 3.3% absolute) error reduction over the vanilla semi-character model. Notably, our pipeline confers robustness on the downstream classifier, outperforming both adversarial training and off-the-shelf spell checkers. Against a BERT model fine-tuned for sentiment analysis, a single adversarially-chosen character attack lowers accuracy from 90.3% to 45.8%. Our defense restores accuracy to 75%\n1 . Surprisingly, better word recognition does not always entail greater robustness. Our analysis reveals that robustness also depends upon a quantity that we denote the sensitivity.", + "full_text": [ + { + "section_name": "Introduction", + "paragraphs": [ + "Despite the rapid progress of deep learning techniques on diverse supervised learning tasks, these models remain brittle to subtle shifts in the data distribution. Even when the permissible changes are confined to barely-perceptible perturbations, training robust models remains an open challenge. Following the discovery that imperceptible attacks could cause image recognition models to misclassify examples BIBREF0 , a veritable sub-field has emerged in which authors iteratively propose attacks and countermeasures.", + "For all the interest in adversarial computer vision, these attacks are rarely encountered outside of academic research. However, adversarial misspellings constitute a longstanding real-world problem. Spammers continually bombard email servers, subtly misspelling words in efforts to evade spam detection while preserving the emails' intended meaning BIBREF1 , BIBREF2 . As another example, programmatic censorship on the Internet has spurred communities to adopt similar methods to communicate surreptitiously BIBREF3 .", + "In this paper, we focus on adversarially-chosen spelling mistakes in the context of text classification, addressing the following attack types: dropping, adding, and swapping internal characters within words. These perturbations are inspired by psycholinguistic studies BIBREF4 , BIBREF5 which demonstrated that humans can comprehend text altered by jumbling internal characters, provided that the first and last characters of each word remain unperturbed.", + "First, in experiments addressing both BiLSTM and fine-tuned BERT models, comprising four different input formats: word-only, char-only, word+char, and word-piece BIBREF6 , we demonstrate that an adversary can degrade a classifier's performance to that achieved by random guessing. This requires altering just two characters per sentence. Such modifications might flip words either to a different word in the vocabulary or, more often, to the out-of-vocabulary token UNK. Consequently, adversarial edits can degrade a word-level model by transforming the informative words to UNK. Intuitively, one might suspect that word-piece and character-level models would be less susceptible to spelling attacks as they can make use of the residual word context. However, our experiments demonstrate that character and word-piece models are in fact more vulnerable. We show that this is due to the adversary's effective capacity for finer grained manipulations on these models. While against a word-level model, the adversary is mostly limited to UNK-ing words, against a word-piece or character-level model, each character-level add, drop, or swap produces a distinct input, providing the adversary with a greater set of options.", + "Second, we evaluate first-line techniques including data augmentation and adversarial training, demonstrating that they offer only marginal benefits here, e.g., a BERT model achieving $90.3$ accuracy on a sentiment classification task, is degraded to $64.1$ by an adversarially-chosen 1-character swap in the sentence, which can only be restored to $69.2$ by adversarial training.", + "Third (our primary contribution), we propose a task-agnostic defense, attaching a word recognition model that predicts each word in a sentence given a full sequence of (possibly misspelled) inputs. The word recognition model's outputs form the input to a downstream classification model. Our word recognition models build upon the RNN-based semi-character word recognition model due to BIBREF7 . While our word recognizers are trained on domain-specific text from the task at hand, they often predict UNK at test time, owing to the small domain-specific vocabulary. To handle unobserved and rare words, we propose several backoff strategies including falling back on a generic word recognizer trained on a larger corpus. Incorporating our defenses, BERT models subject to 1-character attacks are restored to $88.3$ , $81.1$ , $78.0$ accuracy for swap, drop, add attacks respectively, as compared to $69.2$ , $63.6$ , and $50.0$ for adversarial training", + "Fourth, we offer a detailed qualitative analysis, demonstrating that a low word error rate alone is insufficient for a word recognizer to confer robustness on the downstream task. Additionally, we find that it is important that the recognition model supply few degrees of freedom to an attacker. We provide a metric to quantify this notion of sensitivity in word recognition models and study its relation to robustness empirically. Models with low sensitivity and word error rate are most robust." + ] + }, + { + "section_name": "Related Work", + "paragraphs": [ + "Several papers address adversarial attacks on NLP systems. Changes to text, whether word- or character-level, are all perceptible, raising some questions about what should rightly be considered an adversarial example BIBREF8 , BIBREF9 . BIBREF10 address the reading comprehension task, showing that by appending distractor sentences to the end of stories from the SQuAD dataset BIBREF11 , they could cause models to output incorrect answers. Inspired by this work, BIBREF12 demonstrate an attack that breaks entailment systems by replacing a single word with either a synonym or its hypernym. Recently, BIBREF13 investigated the problem of producing natural-seeming adversarial examples, noting that adversarial examples in NLP are often ungrammatical BIBREF14 .", + "In related work on character-level attacks, BIBREF8 , BIBREF15 explored gradient-based methods to generate string edits to fool classification and translation systems, respectively. While their focus is on efficient methods for generating adversaries, ours is on improving the worst case adversarial performance. Similarly, BIBREF9 studied how synthetic and natural noise affects character-level machine translation. They considered structure invariant representations and adversarial training as defenses against such noise. Here, we show that an auxiliary word recognition model, which can be trained on unlabeled data, provides a strong defense.", + "Spelling correction BIBREF16 is often viewed as a sub-task of grammatical error correction BIBREF17 , BIBREF18 . Classic methods rely on a source language model and a noisy channel model to find the most likely correction for a given word BIBREF19 , BIBREF20 . Recently, neural techniques have been applied to the task BIBREF7 , BIBREF21 , which model the context and orthography of the input together. Our work extends the ScRNN model of BIBREF7 ." + ] + }, + { + "section_name": "Robust Word Recognition", + "paragraphs": [ + "To tackle character-level adversarial attacks, we introduce a simple two-stage solution, placing a word recognition model ( $W$ ) before the downstream classifier ( $C$ ). Under this scheme, all inputs are classified by the composed model $C \\circ W$ . This modular approach, with $W$ and $C$ trained separately, offers several benefits: (i) we can deploy the same word recognition model for multiple downstream classification tasks/models; and (ii) we can train the word recognition model with larger unlabeled corpora.", + "Against adversarial mistakes, two important factors govern the robustness of this combined model: $W$ 's accuracy in recognizing misspelled words and $W$ 's sensitivity to adversarial perturbations on the same input. We discuss these aspects in detail below." + ] + }, + { + "section_name": "ScRNN with Backoff", + "paragraphs": [ + "We now describe semi-character RNNs for word recognition, explain their limitations, and suggest techniques to improve them.", + "Inspired by the psycholinguistic studies BIBREF5 , BIBREF4 , BIBREF7 proposed a semi-character based RNN (ScRNN) that processes a sentence of words with misspelled characters, predicting the correct words at each step. Let $s = \\lbrace w_1, w_2, \\dots , w_n\\rbrace $ denote the input sentence, a sequence of constituent words $w_i$ . Each input word ( $w_i$ ) is represented by concatenating (i) a one hot vector of the first character ( $\\mathbf {w_{i1}}$ ); (ii) a one hot representation of the last character ( $\\mathbf {w_{il}}$ , where $l$ is the length of word $w_i$ ); and (iii) a bag of characters representation of the internal characters ( $\\sum _{j=2}^{l-1}\\mathbf {w_{ij}})$ . ScRNN treats the first and the last characters individually, and is agnostic to the ordering of the internal characters. Each word, represented accordingly, is then fed into a BiLSTM cell. At each sequence step, the training target is the correct corresponding word (output dimension equal to vocabulary size), and the model is optimized with cross-entropy loss.", + "While BIBREF7 demonstrate strong word recognition performance, a drawback of their evaluation setup is that they only attack and evaluate on the subset of words that are a part of their training vocabulary. In such a setting, the word recognition performance is unreasonably dependent on the chosen vocabulary size. In principle, one can design models to predict (correctly) only a few chosen words, and ignore the remaining majority and still reach 100% accuracy. For the adversarial setting, rare and unseen words in the wild are particularly critical, as they provide opportunities for the attackers. A reliable word-recognizer should handle these cases gracefully. Below, we explore different ways to back off when the ScRNN predicts UNK (a frequent outcome for rare and unseen words):", + "Pass-through: word-recognizer passes on the (possibly misspelled) word as is.", + "Backoff to neutral word: Alternatively, noting that passing $\\colorbox {gray!20}{\\texttt {UNK}}$ -predicted words through unchanged exposes the downstream model to potentially corrupted text, we consider backing off to a neutral word like `a', which has a similar distribution across classes.", + "Backoff to background model: We also consider falling back upon a more generic word recognition model trained upon a larger, less-specialized corpus whenever the foreground word recognition model predicts UNK. Figure 1 depicts this scenario pictorially.", + "Empirically, we find that the background model (by itself) is less accurate, because of the large number of words it is trained to predict. Thus, it is best to train a precise foreground model on an in-domain corpus and focus on frequent words, and then to resort to a general-purpose background model for rare and unobserved words. Next, we delineate our second consideration for building robust word-recognizers." + ] + }, + { + "section_name": "Model Sensitivity", + "paragraphs": [ + "In computer vision, an important factor determining the success of an adversary is the norm constraint on the perturbations allowed to an image ( $|| \\bf x - \\bf x^{\\prime }||_{\\infty } < \\epsilon $ ). Higher values of $\\epsilon $ lead to a higher chance of mis-classification for at least one $\\bf x^{\\prime }$ . Defense methods such as quantization BIBREF22 and thermometer encoding BIBREF23 try to reduce the space of perturbations available to the adversary by making the model invariant to small changes in the input.", + "In NLP, we often get such invariance for free, e.g., for a word-level model, most of the perturbations produced by our character-level adversary lead to an UNK at its input. If the model is robust to the presence of these UNK tokens, there is little room for an adversary to manipulate it. Character-level models, on the other hand, despite their superior performance in many tasks, do not enjoy such invariance. This characteristic invariance could be exploited by an attacker. Thus, to limit the number of different inputs to the classifier, we wish to reduce the number of distinct word recognition outputs that an attacker can induce, not just the number of words on which the model is “fooled”. We denote this property of a model as its sensitivity.", + "We can quantify this notion for a word recognition system $W$ as the expected number of unique outputs it assigns to a set of adversarial perturbations. Given a sentence $s$ from the set of sentences $\\mathcal {S}$ , let $A(s) = {s_1}^{\\prime } , {s_2}^{\\prime }, \\dots , {s_n}^{\\prime }$ denote the set of $n$ perturbations to it under attack type $A$ , and let $V$ be the function that maps strings to an input representation for the downstream classifier. For a word level model, $V$ would transform sentences to a sequence of word ids, mapping OOV words to the same UNK ID. Whereas, for a char (or word+char, word-piece) model, $V$ would map inputs to a sequence of character IDs. Formally, sensitivity is defined as ", + "$$S_{W,V}^A=\\mathbb {E}_{s}\\left[\\frac{\\#_{u}(V \\circ W({s_1}^{\\prime }), \\dots , V \\circ W({s_n}^{\\prime }))}{n}\\right] ,$$ (Eq. 12) ", + "where $V \\circ W (s_i)$ returns the input representation (of the downstream classifier) for the output string produced by the word-recognizer $W$ using $s_i$ and $\\#_{u}(\\cdot )$ counts the number of unique arguments.", + "Intuitively, we expect a high value of $S_{W, V}^A$ to lead to a lower robustness of the downstream classifier, since the adversary has more degrees of freedom to attack the classifier. Thus, when using word recognition as a defense, it is prudent to design a low sensitivity system with a low error rate. However, as we will demonstrate, there is often a trade-off between sensitivity and error rate." + ] + }, + { + "section_name": "Synthesizing Adversarial Attacks", + "paragraphs": [ + "Suppose we are given a classifier $C: \\mathcal {S} \\rightarrow \\mathcal {Y}$ which maps natural language sentences $s \\in \\mathcal {S}$ to a label from a predefined set $y \\in \\mathcal {Y}$ . An adversary for this classifier is a function $A$ which maps a sentence $s$ to its perturbed versions $\\lbrace s^{\\prime }_1, s^{\\prime }_2, \\ldots , s^{\\prime }_{n}\\rbrace $ such that each $s^{\\prime }_i$ is close to $s$ under some notion of distance between sentences. We define the robustness of classifier $C$ to the adversary $A$ as: ", + "$$R_{C,A} = \\mathbb {E}_s \\left[\\min _{s^{\\prime } \\in A(s)} \\mathbb {1}[C(s^{\\prime }) = y]\\right],$$ (Eq. 14) ", + "where $y$ represents the ground truth label for $s$ . In practice, a real-world adversary may only be able to query the classifier a few times, hence $R_{C,A}$ represents the worst-case adversarial performance of $C$ . Methods for generating adversarial examples, such as HotFlip BIBREF8 , focus on efficient algorithms for searching the $\\min $ above. Improving $R_{C,A}$ would imply better robustness against all these methods.", + "We explore adversaries which perturb sentences with four types of character-level edits:", + "(1) Swap: swapping two adjacent internal characters of a word. (2) Drop: removing an internal character of a word. (3) Keyboard: substituting an internal character with adjacent characters of QWERTY keyboard (4) Add: inserting a new character internally in a word. In line with the psycholinguistic studies BIBREF5 , BIBREF4 , to ensure that the perturbations do not affect human ability to comprehend the sentence, we only allow the adversary to edit the internal characters of a word, and not edit stopwords or words shorter than 4 characters.", + "For 1-character attacks, we try all possible perturbations listed above until we find an adversary that flips the model prediction. For 2-character attacks, we greedily fix the edit which had the least confidence among 1-character attacks, and then try all the allowed perturbations on the remaining words. Higher order attacks can be performed in a similar manner. The greedy strategy reduces the computation required to obtain higher order attacks, but also means that the robustness score is an upper bound on the true robustness of the classifier." + ] + }, + { + "section_name": "Experiments and Results", + "paragraphs": [ + "In this section, we first discuss our experiments on the word recognition systems." + ] + }, + { + "section_name": "Word Error Correction", + "paragraphs": [ + "Data: We evaluate the spell correctors from § \"Robust Word Recognition\" on movie reviews from the Stanford Sentiment Treebank (SST) BIBREF24 . The SST dataset consists of 8544 movie reviews, with a vocabulary of over 16K words. As a background corpus, we use the IMDB movie reviews BIBREF25 , which contain 54K movie reviews, and a vocabulary of over 78K words. The two datasets do not share any reviews in common. The spell-correction models are evaluated on their ability to correct misspellings. The test setting consists of reviews where each word (with length $\\ge 4$ , barring stopwords) is attacked by one of the attack types (from swap, add, drop and keyboard attacks). In the all attack setting, we mix all attacks by randomly choosing one for each word. This most closely resembles a real world attack setting.", + "In addition to our word recognition models, we also compare to After The Deadline (ATD), an open-source spell corrector. We found ATD to be the best freely-available corrector. We refer the reader to BIBREF7 for comparisons of ScRNN to other anonymized commercial spell checkers.", + "For the ScRNN model, we use a single-layer Bi-LSTM with a hidden dimension size of 50. The input representation consists of 198 dimensions, which is thrice the number of unique characters (66) in the vocabulary. We cap the vocabulary size to 10K words, whereas we use the entire vocabulary of 78470 words when we backoff to the background model. For training these networks, we corrupt the movie reviews according to all attack types, i.e., applying one of the 4 attack types to each word, and trying to reconstruct the original words via cross entropy loss.", + "We calculate the word error rates (WER) of each of the models for different attacks and present our findings in Table 2 . Note that ATD incorrectly predicts $11.2$ words for every 100 words (in the `all' setting), whereas, all of the backoff variations of the ScRNN reconstruct better. The most accurate variant involves backing off to the background model, resulting in a low error rate of $6.9\\%$ , leading to the best performance on word recognition. This is a $32\\%$ relative error reduction compared to the vanilla ScRNN model with a pass-through backoff strategy. We can attribute the improved performance to the fact that there are $5.25\\%$ words in the test corpus that are unseen in the training corpus, and are thus only recoverable by backing off to a larger corpus. Notably, only training on the larger background corpus does worse, at $8.7\\%$ , since the distribution of word frequencies is different in the background corpus compared to the foreground corpus." + ] + }, + { + "section_name": "Robustness to adversarial attacks", + "paragraphs": [ + "We use sentiment analysis and paraphrase detection as downstream tasks, as for these two tasks, 1-2 character edits do not change the output labels.", + "For sentiment classification, we systematically study the effect of character-level adversarial attacks on two architectures and four different input formats. The first architecture encodes the input sentence into a sequence of embeddings, which are then sequentially processed by a BiLSTM. The first and last states of the BiLSTM are then used by the softmax layer to predict the sentiment of the input. We consider three input formats for this architecture: (1) Word-only: where the input words are encoded using a lookup table; (2) Char-only: where the input words are encoded using a separate single-layered BiLSTM over their characters; and (3) Word $+$ Char: where the input words are encoded using a concatenation of (1) and (2) .", + "The second architecture uses the fine-tuned BERT model BIBREF26 , with an input format of word-piece tokenization. This model has recently set a new state-of-the-art on several NLP benchmarks, including the sentiment analysis task we consider here. All models are trained and evaluated on the binary version of the sentence-level Stanford Sentiment Treebank BIBREF24 dataset with only positive and negative reviews.", + "We also consider the task of paraphrase detection. Here too, we make use of the fine-tuned BERT BIBREF26 , which is trained and evaluated on the Microsoft Research Paraphrase Corpus (MRPC) BIBREF27 .", + "Two common methods for dealing with adversarial examples include: (1) data augmentation (DA) BIBREF28 ; and (2) adversarial training (Adv) BIBREF29 . In DA, the trained model is fine-tuned after augmenting the training set with an equal number of examples randomly attacked with a 1-character edit. In Adv, the trained model is fine-tuned with additional adversarial examples (selected at random) that produce incorrect predictions from the current-state classifier. The process is repeated iteratively, generating and adding newer adversarial examples from the updated classifier model, until the adversarial accuracy on dev set stops improving.", + "In Table 3 , we examine the robustness of the sentiment models under each attack and defense method. In the absence of any attack or defense, BERT (a word-piece model) performs the best ( $90.3\\%$ ) followed by word+char models ( $80.5\\%$ ), word-only models ( $79.2\\%$ ) and then char-only models ( $70.3\\%$ ). However, even single-character attacks (chosen adversarially) can be catastrophic, resulting in a significantly degraded performance of $46\\%$ , $57\\%$ , $59\\%$ and $33\\%$ , respectively under the `all' setting.", + "Intuitively, one might suppose that word-piece and character-level models would be more robust to such attacks given they can make use of the remaining context. However, we find that they are the more susceptible. To see why, note that the word `beautiful' can only be altered in a few ways for word-only models, either leading to an UNK or an existing vocabulary word, whereas, word-piece and character-only models treat each unique character combination differently. This provides more variations that an attacker can exploit. Following similar reasoning, add and key attacks pose a greater threat than swap and drop attacks. The robustness of different models can be ordered as word-only $>$ word+char $>$ char-only $\\sim $ word-piece, and the efficacy of different attacks as add $>$ key $>$ drop $>$ swap.", + "Next, we scrutinize the effectiveness of defense methods when faced against adversarially chosen attacks. Clearly from table 3 , DA and Adv are not effective in this case. We observed that despite a low training error, these models were not able to generalize to attacks on newer words at test time. ATD spell corrector is the most effective on keyboard attacks, but performs poorly on other attack types, particularly the add attack strategy.", + "The ScRNN model with pass-through backoff offers better protection, bringing back the adversarial accuracy within $5\\%$ range for the swap attack. It is also effective under other attack classes, and can mitigate the adversarial effect in word-piece models by $21\\%$ , character-only models by $19\\%$ , and in word, and word+char models by over $4.5\\%$ . This suggests that the direct training signal of word error correction is more effective than the indirect signal of sentiment classification available to DA and Adv for model robustness.", + "We observe additional gains by using background models as a backoff alternative, because of its lower word error rate (WER), especially, under the swap and drop attacks. However, these gains do not consistently translate in all other settings, as lower WER is necessary but not sufficient. Besides lower error rate, we find that a solid defense should furnish the attacker the fewest options to attack, i.e. it should have a low sensitivity.", + "As we shall see in section § \"Understanding Model Sensitivity\" , the backoff neutral variation has the lowest sensitivity due to mapping UNK predictions to a fixed neutral word. Thus, it results in the highest robustness on most of the attack types for all four model classes.", + "Table 4 shows the accuracy of BERT on 200 examples from the dev set of the MRPC paraphrase detection task under various attack and defense settings. We re-trained the ScRNN model variants on the MRPC training set for these experiments. Again, we find that simple 1-2 character attacks can bring down the accuracy of BERT significantly ( $89\\%$ to $31\\%$ ). Word recognition models can provide an effective defense, with both our pass-through and neutral variants recovering most of the accuracy. While the neutral backoff model is effective on 2-char attacks, it hurts performance in the no attack setting, since it incorrectly modifies certain correctly spelled entity names. Since the two variants are already effective, we did not train a background model for this task." + ] + }, + { + "section_name": "Understanding Model Sensitivity", + "paragraphs": [ + "To study model sensitivity, for each sentence, we perturb one randomly-chosen word and replace it with all possible perturbations under a given attack type. The resulting set of perturbed sentences is then fed to the word recognizer (whose sensitivity is to be estimated). As described in equation 12 , we count the number of unique predictions from the output sentences. Two corrections are considered unique if they are mapped differently by the downstream classifier.", + "The neutral backoff variant has the lowest sensitivity (Table 5 ). This is expected, as it returns a fixed neutral word whenever the ScRNN predicts an UNK, therefore reducing the number of unique outputs it predicts. Open vocabulary (i.e. char-only, word+char, word-piece) downstream classifiers consider every unique combination of characters differently, whereas word-only classifiers internally treat all out of vocabulary (OOV) words alike. Hence, for char-only, word+char, and word-piece models, the pass-through version is more sensitive than the background variant, as it passes words as is (and each combination is considered uniquely). However, for word-only models, pass-through is less sensitive as all the OOV character combinations are rendered identical.", + "Ideally, a preferred defense is one with low sensitivity and word error rate. In practice, however, we see that a low error rate often comes at the cost of sensitivity. We see this trade-off in Figure 2 , where we plot WER and sensitivity on the two axes, and depict the robustness when using different backoff variants. Generally, sensitivity is the more dominant factor out of the two, as the error rates of the considered variants are reasonably low.", + "We verify if the sentiment (of the reviews) is preserved with char-level attacks. In a human study with 50 attacked (and subsequently misclassified), and 50 unchanged reviews, it was noted that 48 and 49, respectively, preserved the sentiment." + ] + }, + { + "section_name": "Conclusion", + "paragraphs": [ + "As character and word-piece inputs become commonplace in modern NLP pipelines, it is worth highlighting the vulnerability they add. We show that minimally-doctored attacks can bring down accuracy of classifiers to random guessing. We recommend word recognition as a safeguard against this and build upon RNN-based semi-character word recognizers. We discover that when used as a defense mechanism, the most accurate word recognition models are not always the most robust against adversarial attacks. Additionally, we highlight the need to control the sensitivity of these models to achieve high robustness." + ] + }, + { + "section_name": "Acknowledgements", + "paragraphs": [ + "The authors are grateful to Graham Neubig, Eduard Hovy, Paul Michel, Mansi Gupta, and Antonios Anastasopoulos for suggestions and feedback." + ] + } + ], + "qas": [ + { + "question": "What does the \"sensitivity\" quantity denote?", + "question_id": "0d9fcc715dee0ec85132b3f4a730d7687b6a06f4", + "nlp_background": "infinity", + "topic_background": "unfamiliar", + "paper_read": "no", + "search_query": "", + "answers": [ + { + "answer": { + "unanswerable": false, + "evidence": [ + "In NLP, we often get such invariance for free, e.g., for a word-level model, most of the perturbations produced by our character-level adversary lead to an UNK at its input. If the model is robust to the presence of these UNK tokens, there is little room for an adversary to manipulate it. Character-level models, on the other hand, despite their superior performance in many tasks, do not enjoy such invariance. This characteristic invariance could be exploited by an attacker. Thus, to limit the number of different inputs to the classifier, we wish to reduce the number of distinct word recognition outputs that an attacker can induce, not just the number of words on which the model is “fooled”. We denote this property of a model as its sensitivity." + ], + "highlighted_evidence": [ + "Thus, to limit the number of different inputs to the classifier, we wish to reduce the number of distinct word recognition outputs that an attacker can induce, not just the number of words on which the model is “fooled”. We denote this property of a model as its sensitivity." + ], + "yes_no": null, + "free_form_answer": "", + "extractive_spans": [ + "the number of distinct word recognition outputs that an attacker can induce" + ] + }, + "annotation_id": "15ddd827360695f7846ceff0ee800b9d874d0538", + "worker_id": "5d0eb97e8e840e171f73b7642c2c89dd3984157b" + }, + { + "answer": { + "unanswerable": false, + "evidence": [ + "In NLP, we often get such invariance for free, e.g., for a word-level model, most of the perturbations produced by our character-level adversary lead to an UNK at its input. If the model is robust to the presence of these UNK tokens, there is little room for an adversary to manipulate it. Character-level models, on the other hand, despite their superior performance in many tasks, do not enjoy such invariance. This characteristic invariance could be exploited by an attacker. Thus, to limit the number of different inputs to the classifier, we wish to reduce the number of distinct word recognition outputs that an attacker can induce, not just the number of words on which the model is “fooled”. We denote this property of a model as its sensitivity.", + "We can quantify this notion for a word recognition system $W$ as the expected number of unique outputs it assigns to a set of adversarial perturbations. Given a sentence $s$ from the set of sentences $\\mathcal {S}$ , let $A(s) = {s_1}^{\\prime } , {s_2}^{\\prime }, \\dots , {s_n}^{\\prime }$ denote the set of $n$ perturbations to it under attack type $A$ , and let $V$ be the function that maps strings to an input representation for the downstream classifier. For a word level model, $V$ would transform sentences to a sequence of word ids, mapping OOV words to the same UNK ID. Whereas, for a char (or word+char, word-piece) model, $V$ would map inputs to a sequence of character IDs. Formally, sensitivity is defined as" + ], + "highlighted_evidence": [ + "We denote this property of a model as its sensitivity.\n\nWe can quantify this notion for a word recognition system $W$ as the expected number of unique outputs it assigns to a set of adversarial perturbations." + ], + "yes_no": null, + "free_form_answer": "The expected number of unique outputs a word recognition system assigns to a set of adversarial perturbations ", + "extractive_spans": [] + }, + "annotation_id": "435b631f0001210641a454d504d67e36ba058568", + "worker_id": "c7d4a630661cd719ea504dba56393f78278b296b" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "the number of distinct word recognition outputs that an attacker can induce, not just the number of words on which the model is “fooled”" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "In NLP, we often get such invariance for free, e.g., for a word-level model, most of the perturbations produced by our character-level adversary lead to an UNK at its input. If the model is robust to the presence of these UNK tokens, there is little room for an adversary to manipulate it. Character-level models, on the other hand, despite their superior performance in many tasks, do not enjoy such invariance. This characteristic invariance could be exploited by an attacker. Thus, to limit the number of different inputs to the classifier, we wish to reduce the number of distinct word recognition outputs that an attacker can induce, not just the number of words on which the model is “fooled”. We denote this property of a model as its sensitivity." + ], + "highlighted_evidence": [ + "Thus, to limit the number of different inputs to the classifier, we wish to reduce the number of distinct word recognition outputs that an attacker can induce, not just the number of words on which the model is “fooled”. We denote this property of a model as its sensitivity." + ] + }, + "annotation_id": "ff3f7fa487bdfae25185c1d3c5deb91bba515715", + "worker_id": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4" + } + ], + "question_writer": "50d8b4a941c26b89482c94ab324b5a274f9ced66" + }, + { + "question": "What end tasks do they evaluate on?", + "question_id": "8910ee2236a497c92324bbbc77c596dba39efe46", + "nlp_background": "infinity", + "topic_background": "unfamiliar", + "paper_read": "no", + "search_query": "", + "answers": [ + { + "answer": { + "unanswerable": false, + "evidence": [ + "For sentiment classification, we systematically study the effect of character-level adversarial attacks on two architectures and four different input formats. The first architecture encodes the input sentence into a sequence of embeddings, which are then sequentially processed by a BiLSTM. The first and last states of the BiLSTM are then used by the softmax layer to predict the sentiment of the input. We consider three input formats for this architecture: (1) Word-only: where the input words are encoded using a lookup table; (2) Char-only: where the input words are encoded using a separate single-layered BiLSTM over their characters; and (3) Word $+$ Char: where the input words are encoded using a concatenation of (1) and (2) .", + "We also consider the task of paraphrase detection. Here too, we make use of the fine-tuned BERT BIBREF26 , which is trained and evaluated on the Microsoft Research Paraphrase Corpus (MRPC) BIBREF27 ." + ], + "highlighted_evidence": [ + "For sentiment classification, we systematically study the effect of character-level adversarial attacks on two architectures and four different input formats. ", + "We also consider the task of paraphrase detection." + ], + "yes_no": null, + "free_form_answer": "Sentiment analysis and paraphrase detection under adversarial attacks", + "extractive_spans": [] + }, + "annotation_id": "051e472956de9c3f0fc4ee775e95c5d66bda0d05", + "worker_id": "c7d4a630661cd719ea504dba56393f78278b296b" + } + ], + "question_writer": "50d8b4a941c26b89482c94ab324b5a274f9ced66" + }, + { + "question": "What is a semicharacter architecture?", + "question_id": "2c59528b6bc5b5dc28a7b69b33594b274908cca6", + "nlp_background": "infinity", + "topic_background": "unfamiliar", + "paper_read": "no", + "search_query": "", + "answers": [ + { + "answer": { + "unanswerable": false, + "evidence": [ + "Inspired by the psycholinguistic studies BIBREF5 , BIBREF4 , BIBREF7 proposed a semi-character based RNN (ScRNN) that processes a sentence of words with misspelled characters, predicting the correct words at each step. Let $s = \\lbrace w_1, w_2, \\dots , w_n\\rbrace $ denote the input sentence, a sequence of constituent words $w_i$ . Each input word ( $w_i$ ) is represented by concatenating (i) a one hot vector of the first character ( $\\mathbf {w_{i1}}$ ); (ii) a one hot representation of the last character ( $\\mathbf {w_{il}}$ , where $l$ is the length of word $w_i$ ); and (iii) a bag of characters representation of the internal characters ( $\\sum _{j=2}^{l-1}\\mathbf {w_{ij}})$ . ScRNN treats the first and the last characters individually, and is agnostic to the ordering of the internal characters. Each word, represented accordingly, is then fed into a BiLSTM cell. At each sequence step, the training target is the correct corresponding word (output dimension equal to vocabulary size), and the model is optimized with cross-entropy loss." + ], + "highlighted_evidence": [ + "Inspired by the psycholinguistic studies BIBREF5 , BIBREF4 , BIBREF7 proposed a semi-character based RNN (ScRNN) that processes a sentence of words with misspelled characters, predicting the correct words at each step. Let $s = \\lbrace w_1, w_2, \\dots , w_n\\rbrace $ denote the input sentence, a sequence of constituent words $w_i$ . Each input word ( $w_i$ ) is represented by concatenating (i) a one hot vector of the first character ( $\\mathbf {w_{i1}}$ ); (ii) a one hot representation of the last character ( $\\mathbf {w_{il}}$ , where $l$ is the length of word $w_i$ ); and (iii) a bag of characters representation of the internal characters ( $\\sum _{j=2}^{l-1}\\mathbf {w_{ij}})$ . ScRNN treats the first and the last characters individually, and is agnostic to the ordering of the internal characters. Each word, represented accordingly, is then fed into a BiLSTM cell. At each sequence step, the training target is the correct corresponding word (output dimension equal to vocabulary size), and the model is optimized with cross-entropy loss." + ], + "yes_no": null, + "free_form_answer": "A semi-character based RNN (ScRNN) treats the first and last characters individually, and is agnostic to the ordering of the internal characters", + "extractive_spans": [] + }, + "annotation_id": "651d1a88bfbe9b4ae67f475766bf0695324c5c35", + "worker_id": "c7d4a630661cd719ea504dba56393f78278b296b" + }, + { + "answer": { + "unanswerable": false, + "extractive_spans": [ + "processes a sentence of words with misspelled characters, predicting the correct words at each step" + ], + "yes_no": null, + "free_form_answer": "", + "evidence": [ + "Third (our primary contribution), we propose a task-agnostic defense, attaching a word recognition model that predicts each word in a sentence given a full sequence of (possibly misspelled) inputs. The word recognition model's outputs form the input to a downstream classification model. Our word recognition models build upon the RNN-based semi-character word recognition model due to BIBREF7 . While our word recognizers are trained on domain-specific text from the task at hand, they often predict UNK at test time, owing to the small domain-specific vocabulary. To handle unobserved and rare words, we propose several backoff strategies including falling back on a generic word recognizer trained on a larger corpus. Incorporating our defenses, BERT models subject to 1-character attacks are restored to $88.3$ , $81.1$ , $78.0$ accuracy for swap, drop, add attacks respectively, as compared to $69.2$ , $63.6$ , and $50.0$ for adversarial training", + "Inspired by the psycholinguistic studies BIBREF5 , BIBREF4 , BIBREF7 proposed a semi-character based RNN (ScRNN) that processes a sentence of words with misspelled characters, predicting the correct words at each step. Let $s = \\lbrace w_1, w_2, \\dots , w_n\\rbrace $ denote the input sentence, a sequence of constituent words $w_i$ . Each input word ( $w_i$ ) is represented by concatenating (i) a one hot vector of the first character ( $\\mathbf {w_{i1}}$ ); (ii) a one hot representation of the last character ( $\\mathbf {w_{il}}$ , where $l$ is the length of word $w_i$ ); and (iii) a bag of characters representation of the internal characters ( $\\sum _{j=2}^{l-1}\\mathbf {w_{ij}})$ . ScRNN treats the first and the last characters individually, and is agnostic to the ordering of the internal characters. Each word, represented accordingly, is then fed into a BiLSTM cell. At each sequence step, the training target is the correct corresponding word (output dimension equal to vocabulary size), and the model is optimized with cross-entropy loss." + ], + "highlighted_evidence": [ + "Our word recognition models build upon the RNN-based semi-character word recognition model due to BIBREF7 .", + "Inspired by the psycholinguistic studies BIBREF5 , BIBREF4 , BIBREF7 proposed a semi-character based RNN (ScRNN) that processes a sentence of words with misspelled characters, predicting the correct words at each step." + ] + }, + "annotation_id": "9a812c2385822cf7d1d38a66a2271d932de79cb8", + "worker_id": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4" + } + ], + "question_writer": "50d8b4a941c26b89482c94ab324b5a274f9ced66" + }, + { + "question": "Do they experiment with offering multiple candidate corrections and voting on the model output, since this seems highly likely to outperform a one-best correction?", + "question_id": "6b367775a081f4d2423dc756c9b65b6eef350345", + "nlp_background": "infinity", + "topic_background": "unfamiliar", + "paper_read": "no", + "search_query": "", + "answers": [ + { + "answer": { + "unanswerable": false, + "evidence": [], + "highlighted_evidence": [], + "yes_no": false, + "free_form_answer": "", + "extractive_spans": [] + }, + "annotation_id": "d04940d8a2d81a44b32ff6bd0872d138dedcd77c", + "worker_id": "c7d4a630661cd719ea504dba56393f78278b296b" + } + ], + "question_writer": "50d8b4a941c26b89482c94ab324b5a274f9ced66" + }, + { + "question": "Why is the adversarial setting appropriate for misspelling recognition?", + "question_id": "bc01853512eb3c11528e33003ceb233d7c1d7038", + "nlp_background": "infinity", + "topic_background": "unfamiliar", + "paper_read": "no", + "search_query": "", + "answers": [ + { + "answer": { + "unanswerable": false, + "evidence": [ + "For all the interest in adversarial computer vision, these attacks are rarely encountered outside of academic research. However, adversarial misspellings constitute a longstanding real-world problem. Spammers continually bombard email servers, subtly misspelling words in efforts to evade spam detection while preserving the emails' intended meaning BIBREF1 , BIBREF2 . As another example, programmatic censorship on the Internet has spurred communities to adopt similar methods to communicate surreptitiously BIBREF3 ." + ], + "highlighted_evidence": [ + "However, adversarial misspellings constitute a longstanding real-world problem. Spammers continually bombard email servers, subtly misspelling words in efforts to evade spam detection while preserving the emails' intended meaning BIBREF1 , BIBREF2 ." + ], + "yes_no": null, + "free_form_answer": "Adversarial misspellings are a real-world problem", + "extractive_spans": [] + }, + "annotation_id": "76dc08d74d886e2608305de45913077f16620a08", + "worker_id": "c7d4a630661cd719ea504dba56393f78278b296b" + } + ], + "question_writer": "50d8b4a941c26b89482c94ab324b5a274f9ced66" + }, + { + "question": "Why do they experiment with RNNs instead of transformers for this task?", + "question_id": "67ec8ef85844e01746c13627090dc2706bb2a4f3", + "nlp_background": "infinity", + "topic_background": "unfamiliar", + "paper_read": "no", + "search_query": "", + "answers": [ + { + "answer": { + "unanswerable": true, + "evidence": [], + "highlighted_evidence": [], + "yes_no": null, + "free_form_answer": "", + "extractive_spans": [] + }, + "annotation_id": "e9f12ab4ced667d482b8b8cb4a3a9e98f2c42b25", + "worker_id": "c7d4a630661cd719ea504dba56393f78278b296b" + } + ], + "question_writer": "50d8b4a941c26b89482c94ab324b5a274f9ced66" + }, + { + "question": "How do the backoff strategies work?", + "question_id": "ba539cab80d25c3e20f39644415ed48b9e4e4185", + "nlp_background": "infinity", + "topic_background": "unfamiliar", + "paper_read": "no", + "search_query": "", + "answers": [ + { + "answer": { + "unanswerable": false, + "extractive_spans": [], + "yes_no": null, + "free_form_answer": "In pass-through, the recognizer passes on the possibly misspelled word, backoff to neutral word backs off to a word with similar distribution across classes and backoff to background model backs off to a more generic word recognition model trained with larger and less specialized corpus.", + "evidence": [ + "While BIBREF7 demonstrate strong word recognition performance, a drawback of their evaluation setup is that they only attack and evaluate on the subset of words that are a part of their training vocabulary. In such a setting, the word recognition performance is unreasonably dependent on the chosen vocabulary size. In principle, one can design models to predict (correctly) only a few chosen words, and ignore the remaining majority and still reach 100% accuracy. For the adversarial setting, rare and unseen words in the wild are particularly critical, as they provide opportunities for the attackers. A reliable word-recognizer should handle these cases gracefully. Below, we explore different ways to back off when the ScRNN predicts UNK (a frequent outcome for rare and unseen words):", + "Pass-through: word-recognizer passes on the (possibly misspelled) word as is.", + "Backoff to neutral word: Alternatively, noting that passing $\\colorbox {gray!20}{\\texttt {UNK}}$ -predicted words through unchanged exposes the downstream model to potentially corrupted text, we consider backing off to a neutral word like `a', which has a similar distribution across classes.", + "Backoff to background model: We also consider falling back upon a more generic word recognition model trained upon a larger, less-specialized corpus whenever the foreground word recognition model predicts UNK. Figure 1 depicts this scenario pictorially." + ], + "highlighted_evidence": [ + "Below, we explore different ways to back off when the ScRNN predicts UNK (a frequent outcome for rare and unseen words):\n\nPass-through: word-recognizer passes on the (possibly misspelled) word as is.\n\nBackoff to neutral word: Alternatively, noting that passing $\\colorbox {gray!20}{\\texttt {UNK}}$ -predicted words through unchanged exposes the downstream model to potentially corrupted text, we consider backing off to a neutral word like `a', which has a similar distribution across classes.\n\nBackoff to background model: We also consider falling back upon a more generic word recognition model trained upon a larger, less-specialized corpus whenever the foreground word recognition model predicts UNK. Figure 1 depicts this scenario pictorially." + ] + }, + "annotation_id": "6b8bfc71d0cc8b871f78fbbfa807d626ddb10b33", + "worker_id": "c1fbdd7a261021041f75fbe00a55b4c386ebbbb4" + }, + { + "answer": { + "unanswerable": false, + "evidence": [ + "While BIBREF7 demonstrate strong word recognition performance, a drawback of their evaluation setup is that they only attack and evaluate on the subset of words that are a part of their training vocabulary. In such a setting, the word recognition performance is unreasonably dependent on the chosen vocabulary size. In principle, one can design models to predict (correctly) only a few chosen words, and ignore the remaining majority and still reach 100% accuracy. For the adversarial setting, rare and unseen words in the wild are particularly critical, as they provide opportunities for the attackers. A reliable word-recognizer should handle these cases gracefully. Below, we explore different ways to back off when the ScRNN predicts UNK (a frequent outcome for rare and unseen words):", + "Pass-through: word-recognizer passes on the (possibly misspelled) word as is.", + "Backoff to neutral word: Alternatively, noting that passing $\\colorbox {gray!20}{\\texttt {UNK}}$ -predicted words through unchanged exposes the downstream model to potentially corrupted text, we consider backing off to a neutral word like `a', which has a similar distribution across classes.", + "Backoff to background model: We also consider falling back upon a more generic word recognition model trained upon a larger, less-specialized corpus whenever the foreground word recognition model predicts UNK. Figure 1 depicts this scenario pictorially." + ], + "highlighted_evidence": [ + "Below, we explore different ways to back off when the ScRNN predicts UNK (a frequent outcome for rare and unseen words):\n\nPass-through: word-recognizer passes on the (possibly misspelled) word as is.\n\nBackoff to neutral word: Alternatively, noting that passing $\\colorbox {gray!20}{\\texttt {UNK}}$ -predicted words through unchanged exposes the downstream model to potentially corrupted text, we consider backing off to a neutral word like `a', which has a similar distribution across classes.\n\nBackoff to background model: We also consider falling back upon a more generic word recognition model trained upon a larger, less-specialized corpus whenever the foreground word recognition model predicts UNK. Figure 1 depicts this scenario pictorially." + ], + "yes_no": null, + "free_form_answer": "Pass-through passes the possibly misspelled word as is, backoff to neutral word backs off to a word with similar distribution across classes and backoff to background model backs off to a more generic word recognition model trained with larger and less specialized corpus.", + "extractive_spans": [] + }, + "annotation_id": "80104d1334e2651bd32ad3e9a568b8c459359027", + "worker_id": "4857c606a55a83454e8d81ffe17e05cf8bc4b75f" + }, + { + "answer": { + "unanswerable": false, + "evidence": [ + "While BIBREF7 demonstrate strong word recognition performance, a drawback of their evaluation setup is that they only attack and evaluate on the subset of words that are a part of their training vocabulary. In such a setting, the word recognition performance is unreasonably dependent on the chosen vocabulary size. In principle, one can design models to predict (correctly) only a few chosen words, and ignore the remaining majority and still reach 100% accuracy. For the adversarial setting, rare and unseen words in the wild are particularly critical, as they provide opportunities for the attackers. A reliable word-recognizer should handle these cases gracefully. Below, we explore different ways to back off when the ScRNN predicts UNK (a frequent outcome for rare and unseen words):", + "Pass-through: word-recognizer passes on the (possibly misspelled) word as is.", + "Backoff to neutral word: Alternatively, noting that passing $\\colorbox {gray!20}{\\texttt {UNK}}$ -predicted words through unchanged exposes the downstream model to potentially corrupted text, we consider backing off to a neutral word like `a', which has a similar distribution across classes.", + "Backoff to background model: We also consider falling back upon a more generic word recognition model trained upon a larger, less-specialized corpus whenever the foreground word recognition model predicts UNK. Figure 1 depicts this scenario pictorially." + ], + "highlighted_evidence": [ + "Below, we explore different ways to back off when the ScRNN predicts UNK (a frequent outcome for rare and unseen words):\n\nPass-through: word-recognizer passes on the (possibly misspelled) word as is.\n\nBackoff to neutral word: Alternatively, noting that passing $\\colorbox {gray!20}{\\texttt {UNK}}$ -predicted words through unchanged exposes the downstream model to potentially corrupted text, we consider backing off to a neutral word like `a', which has a similar distribution across classes.\n\nBackoff to background model: We also consider falling back upon a more generic word recognition model trained upon a larger, less-specialized corpus whenever the foreground word recognition model predicts UNK. Figure 1 depicts this scenario pictorially." + ], + "yes_no": null, + "free_form_answer": "Backoff to \"a\" when an UNK-predicted word is encountered, backoff to a more generic word recognition model when the model predicts UNK", + "extractive_spans": [] + }, + "annotation_id": "cb380adfc6c3f628bd361108352fe8fe82fb0894", + "worker_id": "c7d4a630661cd719ea504dba56393f78278b296b" + } + ], + "question_writer": "50d8b4a941c26b89482c94ab324b5a274f9ced66" + } + ], + "figures_and_tables": [ + { + "file": "1-Table1-1.png", + "caption": "Table 1: Adversarial spelling mistakes inducing sentiment misclassification and word-recognition defenses." + }, + { + "file": "4-Figure1-1.png", + "caption": "Figure 1: A schematic sketch of our proposed word recognition system, consisting of a foreground and a background model. We train the foreground model on the smaller, domain-specific dataset, and the background model on a larger dataset (e.g., the IMDB movie corpus). We train both models to reconstruct the correct word from the orthography and context of the individual words, using synthetically corrupted inputs during training. Subse-" + }, + { + "file": "5-Table2-1.png", + "caption": "Table 2: Word Error Rates (WER) of ScRNN with each backoff strategy, plus ATD and an ScRNN trained only on the background corpus (78K vocabulary) The error rates include 5.25% OOV words." + }, + { + "file": "7-Table3-1.png", + "caption": "Table 3: Accuracy of various classification models, with and without defenses, under adversarial attacks. Even 1-character attacks significantly degrade classifier performance. Our defenses confer robustness, recovering over 76% of the original accuracy, under the ‘all’ setting for all four model classes." + }, + { + "file": "7-Table4-1.png", + "caption": "Table 4: Accuracy of BERT, with and without defenses, on MRPC when attacked under the ‘all’ attack setting." + }, + { + "file": "8-Figure2-1.png", + "caption": "Figure 2: Effect of sensitivity and word error rate on robustness (depicted by the bubble sizes) in word-only models (left) and char-only models (right)." + }, + { + "file": "8-Table5-1.png", + "caption": "Table 5: Sensitivity values for word recognizers. Neutral backoff shows lowest sensitivity." + } + ] + } +} \ No newline at end of file diff --git a/benchmark/RAG/datasets/Benchmark_Lite/SyllabusQA/syllabi/151 syllabus fall 2022 Rounds.docx b/benchmark/RAG/datasets/Benchmark_Lite/SyllabusQA/syllabi/151 syllabus fall 2022 Rounds.docx new file mode 100644 index 00000000..55e85f6c Binary files /dev/null and b/benchmark/RAG/datasets/Benchmark_Lite/SyllabusQA/syllabi/151 syllabus fall 2022 Rounds.docx differ diff --git a/benchmark/RAG/datasets/Benchmark_Lite/SyllabusQA/syllabi/Most Recent P132 Syllabus - Spring 2022.docx b/benchmark/RAG/datasets/Benchmark_Lite/SyllabusQA/syllabi/Most Recent P132 Syllabus - Spring 2022.docx new file mode 100644 index 00000000..04876c84 Binary files /dev/null and b/benchmark/RAG/datasets/Benchmark_Lite/SyllabusQA/syllabi/Most Recent P132 Syllabus - Spring 2022.docx differ diff --git a/benchmark/RAG/datasets/Benchmark_Lite/SyllabusQA/syllabi/ResEcon-213-Fall-2022.docx b/benchmark/RAG/datasets/Benchmark_Lite/SyllabusQA/syllabi/ResEcon-213-Fall-2022.docx new file mode 100644 index 00000000..8636cbb6 Binary files /dev/null and b/benchmark/RAG/datasets/Benchmark_Lite/SyllabusQA/syllabi/ResEcon-213-Fall-2022.docx differ diff --git a/benchmark/RAG/datasets/Benchmark_Lite/SyllabusQA/syllabi/Syllabus SPP 697 PM 2023 (clean).docx b/benchmark/RAG/datasets/Benchmark_Lite/SyllabusQA/syllabi/Syllabus SPP 697 PM 2023 (clean).docx new file mode 100644 index 00000000..76abea12 Binary files /dev/null and b/benchmark/RAG/datasets/Benchmark_Lite/SyllabusQA/syllabi/Syllabus SPP 697 PM 2023 (clean).docx differ diff --git a/benchmark/RAG/datasets/Benchmark_Lite/SyllabusQA/syllabi/Syllabus_Marine_Microbiome_2022_redacted.docx b/benchmark/RAG/datasets/Benchmark_Lite/SyllabusQA/syllabi/Syllabus_Marine_Microbiome_2022_redacted.docx new file mode 100644 index 00000000..8d669d0a Binary files /dev/null and b/benchmark/RAG/datasets/Benchmark_Lite/SyllabusQA/syllabi/Syllabus_Marine_Microbiome_2022_redacted.docx differ diff --git a/benchmark/RAG/datasets/Benchmark_Lite/SyllabusQA/syllabi/advanced_system_software_design_redacted.docx b/benchmark/RAG/datasets/Benchmark_Lite/SyllabusQA/syllabi/advanced_system_software_design_redacted.docx new file mode 100644 index 00000000..2a9ffd08 Binary files /dev/null and b/benchmark/RAG/datasets/Benchmark_Lite/SyllabusQA/syllabi/advanced_system_software_design_redacted.docx differ diff --git a/benchmark/RAG/datasets/Benchmark_Lite/SyllabusQA/syllabi/cs466_sp21_info_redacted.docx b/benchmark/RAG/datasets/Benchmark_Lite/SyllabusQA/syllabi/cs466_sp21_info_redacted.docx new file mode 100644 index 00000000..9ff18eed Binary files /dev/null and b/benchmark/RAG/datasets/Benchmark_Lite/SyllabusQA/syllabi/cs466_sp21_info_redacted.docx differ diff --git a/benchmark/RAG/datasets/Benchmark_Lite/SyllabusQA/syllabi/web_syllabus_4670.docx b/benchmark/RAG/datasets/Benchmark_Lite/SyllabusQA/syllabi/web_syllabus_4670.docx new file mode 100644 index 00000000..3280ecee Binary files /dev/null and b/benchmark/RAG/datasets/Benchmark_Lite/SyllabusQA/syllabi/web_syllabus_4670.docx differ diff --git a/benchmark/RAG/datasets/Benchmark_Lite/SyllabusQA/test.csv b/benchmark/RAG/datasets/Benchmark_Lite/SyllabusQA/test.csv new file mode 100644 index 00000000..1272050c --- /dev/null +++ b/benchmark/RAG/datasets/Benchmark_Lite/SyllabusQA/test.csv @@ -0,0 +1,292 @@ +id,syllabus_name,question_type,question,answer,answer_span_1,answer_span_2,answer_span_3,answer_span_4,answer_span_5,reasoning_step_1,reasoning_step_2,reasoning_step_3,reasoning_step_4,reasoning_step_5 +wx7GzHFTdOzodNWV,cs466_sp21_info_redacted,single factual,How many homework assignments does this course provide?,The course will include six homework assignments.,Homework assignments: 6,,,,,,,,, +qC4sbV9aIWWeGojT,web_syllabus_4670,yes/no,Is there a textbook?,No,"Textbook(s) and/or Other Required Materials: +• None. Lecture notes will be provided.",,,,,,,,, +mYaegEno6GUFUoQF,ResEcon-213-Fall-2022,multi factual,What items are required for this class?,You will need the textboook Applied Statistics in Business & Economics 7th edition. The authors are David P Doane and Lori E Seward. You will also need an iClicker2 and access to a computer.,"Text: Applied Statistics in Business & Economics by David P. Doane and Lori E. Seward, 7th edition.",Personal Response Transmitter (iClicker2),Computer Access:,,,,,,, +GdD0H5ujWygaPdsR,advanced_system_software_design_redacted,single factual,What is the minimum score required to pass the course?,The minimum score to receive a D- in the course is 60.00.,Final Grade Distribution: 60.00 - 62.99 D-,,,,,,,,, +gYaYnrBUUcz7eRpO,web_syllabus_4670,single reasoning,If I failed ECE 3100 can I still take this course?,"No, you cannot take this course if you failed ECE 3100.",,,,,,"The syllabus states a prerequisite of the course is ECE 3100 so in order to take this class, the student must pass ECE 3100.",,,, +jkoctoeFl2comEY4,ResEcon-213-Fall-2022,single reasoning,I have never taken any college class. Is this class for me?,"Due to the prerequisite classes, if you have never taken a college course, you should not take this class.",,,,,,"There are prereq. courses for this class, so no.",,,, +5B2HKnSL2aRvaopd,ResEcon-213-Fall-2022,single factual,What is the text book requirement for the course?,"The text book is Applied Statistics in Business, 7th Edition. with authors David P. Doane, and Lori E. Seward.","Text: Applied Statistics in Business & Economics by David P. Doane and Lori E. Seward, 7th edition",,,,,,,,, +hrJxKUHNYzG6L21N,cs466_sp21_info_redacted,summarization,What kinds of things constitute academic dishonesty?,"If a student is found to be cheating on any test, plagiarizing homework, fabricating homework, or facilitating dishonesty in any way, they are guilty of academic dishonesty.",Academic dishonesty includes but is not limited to: cheating,plagiarism ,facilitating dishonesty,fabrication,,,,,, +bghxCDPJjMPtZb20,Most Recent P132 Syllabus - Spring 2022,single reasoning,Is Physics 131 a prerequisite for this class?,"Yes, this class builds upon the Physics 131 class.",,,,,,"While not stated so directly on the syllabus, it does make clear that concepts from 131 will be developed in this class.",,,, +DTjmDhUp1j1NpLCV,web_syllabus_4670,summarization,Can you provide a bit more detail on what the labs will entail?,"There are a total of five labs in the course, each centered around using MATLAB. The first two labs will help you to get familiarity with the hardware setup. From there, labs will grow in complexity. In the remainder of labs, students will design communication systems and basic communication systems, and vote on the best performances.","There are five labs, which form the centerpiece of the course. Through these labs, students will develop a working MATLAB -based communication system using PC soundcards. ",The first two labs are designed to acclimate students with the hardware setup that we will be using,"The third lab +involves designing a communication system for a simple channel simulated in software. ","The fourth lab involves designing a basic communication system for the actual, hardware-based channel and +submitting a related “standard” that one could use for designing compatible transmitters and receivers +for this channel. ","In the fifth lab, students will design two communication systems, one that complies with the standard and one that +aims for the highest performance.",,,,, +Sj2XO5jFfuj8IEuH,Syllabus_Marine_Microbiome_2022_redacted,yes/no,"Can you get ""Environmental Microbiology -- from genomes to biogeochemistry"" from the UofSC library?",Yes,"Under Section 2. Required Textbook, it states that yes, this textbook is available through the UofSC library",,,,,,,,, +dPjGq0WFHFqkKJmg,web_syllabus_4670,multi reasoning,What is the total lecture time of this course?,"There are roughly 1,000 minutes of lectures, or just over 16.5 hours, as part of this course.",,,,,,There are two lectures each week,Each lecture is 75 minutes,There are 150 minutes of lectures per week,There are 7 weeks of courses in the syllabus ,"This is 1,000 minutes " +LErkT9RiRXHN1DRk,151 syllabus fall 2022 Rounds,multi reasoning,If student register with the iclicker cloud then will they get full credit ?,Student need to attend 75% of questions to get credit.,,,,,,Clicker is used for grading of participation.,It was used to ensure the participation.,If the student answer 75% of the question then they will get full credit.,, +mRJuRgVu86qParnr,Most Recent P132 Syllabus - Spring 2022,summarization,Why would I want to be on a team and what responsibilities do I have if I'm on one?,"Students that elect to be on teams have much higher average exam scores than students who elect to go solo. When you're on a team, you can be confident that someone you know will always be there to help you with in-class activities and collaborative sections of the exam. When you're on a team, you have a responsibility to be present in class for your teammates. Teams will evaluate each other on their contributions, which results in a multiplier being applied to team quizzes and exams, so it's important to do your best for when you're on a team.","you can +be confident that you will have people to help you on in-class activities. ","In this course, all quizzes and exams will have collaborative portions. If you are on a +team, you will work on these with your team. From working together you will be well +aware of each others’ strengths and can expect a level of trust that people who are not +in a team. Last year, students who elected to be on teams had an team-exam average of +98.3% while those who elected to go solo had an team-exam average of 86.2%","When you sign up to be on an organized team, you are committing to being present in +class for your team mates.","Your team will evaluate you on your contributions using the CATME system. The +resulting multiplier will be applied to your team quizzes and exams. +",,,,,, +ecJA8EwEWBaJsh8l,Syllabus SPP 697 PM 2023 (clean),yes/no,Will the weekly online sessions be more than 2 hours long?,Yes,"Weekly live, on-line sessions are 2.5 hours in length and will be broken into two segments, with a break about halfway through.",,,,,,,,, +hWTLB4nc6rUzKHjm,ResEcon-213-Fall-2022,multi reasoning,"If you miss class and don't turn in homework, by how much will your score be affected?","If you miss class and are late turning in homework, your grade will be impacted by up to 15%",,,,,,Attendance counts for 3% of your grade,Homework is 12% of your final grade,,, +J1coidzFy5dMeSbw,cs466_sp21_info_redacted,single reasoning,If I passed COMPSCI 311 but struggled with it can I enroll in this course?,"COMPSCI 311 is the prerequisite, but since this course relies heavily on mathematical maturity, if you did not master the prerequisite, you will likely struggle in this course even though you qualify for enrollment.++++",,,,,,If you earned less than a B in COMPSCI 311 you should consider retaking it.,,,, +Jm5YJM6SOFEkqiVv,web_syllabus_4670,summarization,What is the point of this class?,"The goal of this course is to help student able to understand how communication systems are designed and why they are designed in the way they are. This course covers communication theory. Students will design a working audio-band communication system that relies on OFDM. After the course is finished students will have an appreciation of the importance of standardization in commercial communication systems design, the attention to detail necessary to write a successful standard and the exactitude required to implement a standard properly.","Catalog Description: +The goal of this course is to understand how state-of-the-art digital communication systems are +designed and why they are designed in the way they are. The course will cover communication theory, +transceiver algorithms that enable reliable communication, wireless channels, and modern +communication standards (such as 3GPP LTE and WiFi). The students will design a working audio-band +communication system that relies on orthogonal frequency-division multiplexing (OFDM)","Student Outcomes [ABET]: +• +Attain an ability to compute the MAP and ML detection rules for a given detection problem and a +recognition of when to apply them +• +Understand how to model communication channels mathematically, including both deterministic +and stochastic impairments and the physical phenomena that give rise to these impairments +• +Attain an ability to optimally allocate bit-rate and power over a vector Gaussian channel with +varying signal-to-noise ratios among its components +• +Attain an ability to design a working OFDM system over a supplied audio-band channel +• +Develop an appreciation of the importance of standardization in commercial communication +systems design, the attention-to-detail necessary to write a successful standard, and the exactitude +required to implement a standard properly",,,,,,,, +4Dpygp03rGLdU0Fu,cs466_sp21_info_redacted,multi reasoning,What do we need to do on our homework to make sure we get the points? ,"You can work with others, but you must make sure your solutions are shown and are readable. ",,,,,,You can work with other folks,You must do your final solution. ,You must show your work. ,it must be able to be read you will get a 0 if not readable. , +y7sU7ouuoBG2Dc5b,Most Recent P132 Syllabus - Spring 2022,single factual,When can I do the homework?,As soon as you complete the syllabus quiz.,"All of the homework is accessible as soon as you complete the syllabus quiz. Given that +the assignments are preparatory, you do not need to wait for us to finish Unit I before +starting the Unit II homework (or even the Unit III!). T",,,,,,,,, +GW4HAGJof69VaJeF,advanced_system_software_design_redacted,yes/no,Is there a text book required for this course?,Yes,"Recommended Text Books +OSTEP: http://pages.cs.wisc.edu/∼remzi/OSTEP/",,,,,,,,, +QQcPIAt4WLH6ngt8,Most Recent P132 Syllabus - Spring 2022,multi factual,What does the module extensively used for?,"Modules can be used for taking surveys, to get extra credit opportunities, posting slides and other resources."," I will use Moodle extensively for: +● Hub for all other tools"," +● Surveys"," +● Posting slides and other resources +",● Extra credit opportunities.,,,,,, +ELg8HnTT34jzgF1D,cs466_sp21_info_redacted,summarization,What is the nature of the homework for this course?,"There are six assignments, which must be hand written in pencil by you. You do not need to show your work and can collaborate with others so long as you name them. There is no programming code required in the homeworks but LATEX is recommended.",There will be six homework assignments worth 100 points each.,"The assignments will +be pencil-and-paper; there is no programming required. "," For the homeworks, you can work with one +another as long as you explicitly list your collaborators for each problem."," Additionally, you must +write your final solutions by yourself, as if you are taking your exam.",". I will not take into account any material other than your written solution, +such as program code, in your grade. It is highly recommended that you typeset your solutions in +LATEX.",,,,, +5GPJry0NyokwqKy7,cs466_sp21_info_redacted,single factual,Is there a required textbook?,There is a textbook for the course but it's optional.,An optional accompanying textbook is available,,,,,,,,, +HqIiZj0eUlt6X6KA,cs466_sp21_info_redacted,yes/no,"Are you eligible to file for Disability Services if you do not have a documented physical, psychological, or learning disability?",No,"If you have a documented physical, psychological, or learning disability on file with Disability Services (DS), you may be eligible for reasonable +academic accommodations to help you succeed in this course.",,,,,,,,, +YG8XsSOdu5ChFj9I,151 syllabus fall 2022 Rounds,multi reasoning,"If I had an 88% average before the final exam, and my two mid-term grades were 95% and 81%, could I still get an A- in the class?","You would need to score a 93, or higher, on the final in order to get an A- for the class.",,,,,,The minimum score required for an A- is 90%,Scoring a 93% on the final would mean the 81% mid-term score is replaced by the 93%. That would change the mid-term average from 88% to 90.5%,"The 93% final score would bring the grade for the whole class up to a 90.2%, which is an A- ",, +KsUB5btQ3EHceSpx,cs466_sp21_info_redacted,multi factual,How many points are the homework and the exams worth each?,"The homework is worth 100 points each, the midterm is work 100 points, and the final exam is worth 200 points.","There will be comprehensive (up to that point in the class) 100 points midterm as well +as a comprehensive (across the entire course) 200 points final exam.",There will be six homework assignments worth 100 points each.,,,,,,,, +DH7pS1b9T0eEMAs5,Most Recent P132 Syllabus - Spring 2022,single factual,Where does they encourage to post the questions on the forum?,Discord server,"Rather than emailing questions to the teaching staff, I encourage +you to post your questions on the Discord server in order to get a more rapid response.",,,,,,,,, +3CeDzQjNmqGCgdCL,ResEcon-213-Fall-2022,single reasoning,"What makes up the largest portion, or percentage, of the students final grade?",The final exam,,,,,,There are various different graded work types for the class. The largest being the final exam.,,,, +qcpBp8zxlFwchN8T,Most Recent P132 Syllabus - Spring 2022,summarization,When should i start the homework?,Home works can be accessible once you complete the quiz. Can start to write any homework even before we complete the units.,All of the homework is accessible as soon as you complete the syllabus quiz.," Given that the assignments are preparatory, you do not need to wait for us to finish Unit I before starting the Unit II homework (or even the Unit III!). ","The assignments are long, I +encourage you to start early! +",,,,,,, +huMoQUWQejRxuPaH,web_syllabus_4670,single reasoning,Will I miss out on anything if I don't keep up to date on the syllabus until after the class starts?,"Since the syllabus is subject to change, its possible you may miss out on a change that could impact what you were expecting to learn or need to have prepared before taking this course.",,,,,,"If the syllabus is subject to change, the author could add or remove something from the syllabus that may be important to know before hand.",,,, +W7npizUDJGi5SlZY,Most Recent P132 Syllabus - Spring 2022,multi reasoning,I won't be able to turn in my homework on Feb 2. How will this affect my homework grade? ,"Assuming a perfect score on the rest of the homework assignments, the best one can get for the homework portion of the grade is 96%.",,,,,,"Late homework is, at best, 80%.",There are 5 homework assignments. ,480/5=96%,, +AlNFdaLg9zx5DwVz,ResEcon-213-Fall-2022,multi reasoning,I'm looking for a light course load with no homework and only one test. is this class for me?,"Because of the amount of assignments, quizzes, homework and tests, this class is too much for you.",,,,,,There are many assignments.,There is homework,There are many tests.,, +3l8DLZZzCWps12Zv,151 syllabus fall 2022 Rounds,single reasoning,I will not be able to attend class on October the 6th. What will be covered on October the 6th?,"The first exam is held on October 6. If you tell the professor ahead of time, and have a valid excuse, they will drop the exam. ",,,,,,The first exam is held on October the 6th.,,,, +087TCE0iD4QZQswU,151 syllabus fall 2022 Rounds,yes/no,Is Discord required for the class?,Yes,"In the context of this class you will use Discord to get answers to questions about the +class and the content. You can also use it to collaborate with others in class on homework +and quizzes",,,,,,,,, +vQ3nVPr2sneIUpWM,ResEcon-213-Fall-2022,multi factual,What will you need to use your iClicker2 transmitter in class?,"To use your iClicker2 transmitter in class, you must know your SPIRE ID and your iClicker2 ID and link them online using Moodle.",You must link your Spire ID to your iClicker2 transmitter on the Moodle site for this course.,"Each transmitter has a unique ID which you must +link to your SPIRE ID on Moodle.",,,,,,,, +iFCJHEMK12fE4lWZ,advanced_system_software_design_redacted,yes/no,Is there a policy on pronouns and names that the professor uses both online and in class? ,Yes,"Students can indicate their preferred/chosen first name and pronouns on SPIRE, which appear on +class rosters. Please let me know what name and pronouns I should use for you if they are not on +the roster. ",,,,,,,,, +LbQ0f6iefLTlYgIW,Syllabus SPP 697 PM 2023 (clean),single reasoning,Will I get a C grade if I score 83%?,"No, you will get a B.",,,,,,A score of 81-84% is worth a B.,,,, +6m0FiOmHjwgwcyv9,151 syllabus fall 2022 Rounds,multi reasoning,"If I get a 95 on the final exam, a 60 on one mid-term exam, and no score on the other mid-term which I gave a valid excuse for missing, will my overall exam score average out to above 80?","No, your exam scores will average out to 77.5. The scores that will count are your final exam of 95 and your mid-term score of 60.",,,,,,Missed exams with a valid excuse are dropped instead of scored zero.,Final exams can replace a lower scoring mid-term only if both mid-terms are taken.,"Therefore, your exam scores are 95, 60 and N/A, giving an average of 77.5.",, +7QmRywAskqPPyRxs,cs466_sp21_info_redacted,yes/no,Will a students lowest homework score in the class be dropped to make up for excused absences?,Yes,"There will be no makeups on homework. To allow for excused absences, I will drop your lowest homework score.",,,,,,,,, +FAlnQWSM4YKsSV9t,Syllabus_Marine_Microbiome_2022_redacted,summarization,What are the class expectations?,"Students are expected to attend class faithfully, not use their cell phone, be prepared by doing assigned readings and take thorough notes in class. Students should also treat each other and their items with respect. ","attend every class period (prior approval or doctors excuse required for absences) +• come to class prepared to contribute and learn +• do assigned readings before class and take notes during class +• all cell phones should only be used for class-related activities"," respect the dignity, rights, +and property of others",,,,,,,, +JWf88JNOmWfiFZzi,Syllabus SPP 697 PM 2023 (clean),multi factual,How many parts are each session?,Each session has two parts.,"Part 1: Instructor Presentation - Course Orientation and Introductions +Part 2: Instructor Lecture, followed by Class Discussion - Evolution of the Performance +Management Movement in the U.S.","Part 1: Instructor Lecture and Individual Student Presentations on Creating a Performance +Context: +• Instructor Overview Lecture on the 4 Performance Context themes: +o Strategic Outcome Indicators, Strategic Foresight, Strategic Planning, and Enterprise +Risk Management +• Individual Student Presentations on Creating a Performance Context: +o Key National Indicators Presenter: xxxxxxxx +Her presentations will be followed by class discussion of readings associated with +Strategic Outcome Indicators, Strategic Foresight, Strategic Planning, and Enterprise Risk +Management +Part 2: Guest Speaker: xxxxxxxx, acting director, Baltimore Neighborhood Indicators Alliance; +followed by discussion of Team Assignment Memos, Timelines","Part 1: Individual Student Presentations on Alternatives for Informing a Performance +Framework +• Logic Model Presenter: xxxxxxxx +• Lean Six Sigma Presenter: xxxxxxxx +• Balanced Scorecard Presenter: xxxxxxxx +• Baldrige Criteria Presenter: xxxxxxxx +Part 2: Instructor Lecture on “Performance Management Frameworks” followed by Class +discussion of reading assignments and his presentation","Part 1: Instructor Lecture on Performance Measures followed by Class discussion of reading +assignments and his presentation. +Part 2: Guest Speaker: xxxxxxxx Principal Program Analyst, North Carolina Office of State Budget +and Management +","Part 1: +• Instructor Lecture: What is “Theory of Change?” +• Individual Student Presentations on Alternatives for Informing Decisions +o Performance Budgeting/Outcome- or Results-Based Budgeting +Presenter: xxxxxxxx +o Activity Based Costing Presenter: xxxxxxxx +o Program Assessment Rating Tool Presenter: xxxxxxxx +o Agency Strategic Reviews Presenter: xxxxxxxx +Part 2: Guest speaker: xxxxxxxx, former Performance & Innovation Officer, Albuquerque, NM; +Associate, Results for America",,,,, +jsXh45VosCM5bfYb,151 syllabus fall 2022 Rounds,multi factual,How are grades broken down in this course?,"Grading for this course is as follows: there are two exams each worth 20 percent, a final exam worth 24 percent, discussion problems sets worth 20%, Wednesday quizzes worth 15 percent, and The iClicker Cloud attendance is worth 1 percent. ",2 Exams 40 % (20% each),1 Final Exam 24%,Discussion problem sets 20%,Wednesday Quizzes 15%,iClicker Cloud 1% for attendance,,,,, +eShtA8bIQConNJpv,Syllabus SPP 697 PM 2023 (clean),multi reasoning,How can I get the best score possible?,Do well in each of the areas listed for grading.,,,,,,Participation 30%,Service as Co-Moderator 10%,Preparation of two memos 30%,Team contributions 30%, +DYmyXJmH9Y7bdFa7,151 syllabus fall 2022 Rounds,multi reasoning,"If I fail my first exam, Is it possible to still get an A average on all three exams?","As long as you attend all exams, and do well on the second exam and final exam it is still possible to get an A average. Your final exam grade will replace your lowest grade. ",,,,,,"If you score higher on your final exam than one of your regular exams, the lower score will be replaced with the higher. You must attend all exams, and not have gotten a zero from non-attendance. ","If you score a 10 on the first exam, a 93 on the second, and a 95 on the third, the 10 will be replaced with a 95 and the average will be an A. ",,, +KlQu4EPIurT2Tl9Z,Syllabus_Marine_Microbiome_2022_redacted,single factual,How many in-class oral presentations will each student be required to give for this class?,Each student is required to give three in-class oral presentations on assigned topics.,"Under 4. Course Requirements, it states that each student will give three in-class oral presentations.",,,,,,,,, +J0lsYfhU9r5uz7zJ,Syllabus_Marine_Microbiome_2022_redacted,single reasoning,Do you need to be a civilized person to go to school here?,I know there are many things listed and unlisted that will enable students to be good sgtudents. ,,,,,,There is many different things listed and others that students can do to keep with good student policy. ,,,, +9mP1xwX73pnrc0Pk,web_syllabus_4670,yes/no,Does this course have labs?,Yes,There are five labs,,,,,,,,, +TtR5fKILjjNmSwEc,cs466_sp21_info_redacted,summarization,what is the accommodation statement provided by the university?,The statements said that providing equal opportunity for all students. Learning disability on file with disability services are eligible for reasonable academic accommodations,Providing an equal educational opportunity for all students.,Learning disability on file with disability services are eligible for reasonable academic accommodations,,,,,,,, +fPz4HKPX83af2wud,Syllabus SPP 697 PM 2023 (clean),yes/no,Will the written and class presentations be done on Zoom?,Yes,"Written and class presentation assignments – both individual and team, via Zoom",,,,,,,,, +vWcYu1ylFiXFuU5O,Syllabus_Marine_Microbiome_2022_redacted,multi factual,How are the final grades calculated and with what weight? ,"50 % Course Project +20 % Participation in Class +30% Oral Presentations","Final grades will be calculated using the following weighting: +","50% Course project +20% Participation in class discussions +30% Oral presentations",,,,,,,, +U4vV7lDwRYblCPho,web_syllabus_4670,multi reasoning,What does the coursework look like?,The coursework for the course involve attending lecture twice a week for 75 minutes each along with recitations once a week. Weekly assignments are given along with two exams. Weekly assignment can be worked on with other students. There are five labs which are the main focus of the course. The first three labs are done alone while the last two can be done with a partner.,,,,,,"Class and Laboratory Schedule: Lectures: Two 75 min lectures per week Recitations: One per week, which meets occassionally.","Assignments, Exams and Projects: Homework: Weekly assignments. Approximately seven homework assignments per semester Collaboration with students is encouraged. Exams. Two preliminary exams. ","Design Projects: There are five labs, which form the centerpiece of the course.",The first three labs must be done individually. The fourth and fifth labs my be done individually or in pairs., +UaiYr5DAXJx3UxvR,cs466_sp21_info_redacted,yes/no,Is there any makeups for homework?,No,There will be no makeups on homework,,,,,,,,, +G2P0xtAhk5ZUzpKR,Syllabus_Marine_Microbiome_2022_redacted,yes/no,"Is ""Environmental Microbiology -- from genomes to biogeochemistry"" a required textbook for this course?",Yes,Section 2. Required Textbook: -- This is the only required textbook listed for the class. ,,,,,,,,, +eg8gVHx5vik8IpmQ,web_syllabus_4670,multi factual,How will homework be assigned during this course?,Around seven homework assignment will be assigned through out the course and on a weekly basis.,Weekly assignments. ,Approximately seven homework assignments per semester. ,,,,,,,, +Ofrtdv8ONtP31ueL,ResEcon-213-Fall-2022,multi reasoning,What is the highest letter grade that can be achieved without having access to Connect?,The highest letter grade that can be achieved without Connect access is a B-.,,,,,,Connect homework is worth 12% of the final grade,Connect quizzes are worth 13% of the final grade.,The highest percentage that can be achieved without either Connect score is 100-12-13=75%.,75% equates to a B- on the reference guide., +APCF4PL6ogUAeKWj,cs466_sp21_info_redacted,summarization,Can we access the modifications to the course slides prior to each lecture?,The modified slides will be presented and annotated during each class.,"We will follow slides by Mihir Bellare available at https://cseweb.ucsd.edu/ mihir/cse107/slides.html; +however, I will modify them somewhat to suit our needs.",These modified slides will be presented in class., I will annotate my slides during class.,,,,,,, +ENoirQGBll7cHVe4,ResEcon-213-Fall-2022,multi factual,What if I have questions?,"Before coming to me with questions during office hours, first check the syllabus, then check Moodle, and then email the TA.","If you have questions, do the following steps in order: +1) Check the syllabus","2) Check Moodle, slides on Moodle, and announcements","3) Email the Teaching Assistant (include ""ResEc213"" in the subject line)",4) Come to office hours or ask Prof. <PROF_LAST_NAME> before/after class,,,,,, +uHztqyC8W2V4MHDJ,Syllabus_Marine_Microbiome_2022_redacted,multi factual,What parts of the class factor into the final grade? ,"After grades for the Course Project and Oral Presentations are calculated, class participation in discussions is factored in.","Final grades will be calculated using the following weighting: +50% Course project",Participation in class discussions,Oral presentations,,,,,,, +9MgUXhgJEaXSfsDA,advanced_system_software_design_redacted,single reasoning,"If I miss some classes and my attendance portion of my grade suffers, can I make up those missed points? ",Yes - attendance is 5% of the course grade but there are a possible 9% in bonus points available that may allow for making up those missing attendance points. ,,,,,,There is a 5% portion of the final grade for attendance and there is a possible 9% in bonus points that may be available so it is possible to make up missing attendance points with the bonus points. ,,,, +z137LsoCcjdWyvsA,web_syllabus_4670,single factual,How big of an impact are the weekly homework assignments on the final grade?,The weekly homework assignments compose of 15% of the final course grade.,"Course Grading Scheme: (for ECE 4670): 15% Homework assignments, 50% Labs, 35% Prelims",,,,,,,,, +CJYTUGTlprJLXEhj,151 syllabus fall 2022 Rounds,single factual,What kind of support is available for students who are struggling to understand material for an upcoming quiz?,Quiz help sessions conducted by the Tas are offered following class and on evenings.,"There will be a quiz help sessions run by our Tas after class +and in the evening.",,,,,,,,, +rLFj9ykGI3jW23QQ,advanced_system_software_design_redacted,single factual,How much does attendance count towards the final grade? ,Attendance is 5% of your final grade. ,Attendance 5%,,,,,,,,, +g2YBtyb7XFU2Gcvq,web_syllabus_4670,single factual,How many credits is this course worth?,This course is worth 4 credits.,"Credit Hours: 4 hours +",,,,,,,,, +hghUKDxoAEPv6eX6,Most Recent P132 Syllabus - Spring 2022,summarization,What are some ways I can get help during the class?,"Students can meet with the professor at office hours or by appointment, visit the Physics Consultation Room at Hasbrouck Hall, or post a question of the Discord server.","Thus, if my office hours do not work for you, +then do not hesitate to reach out and we will arrange an appointment. ","Hasbrouck 115 is the physics consultation room. This +room will be staffed by a physics graduate student +throughout the day. ","Basically, most +of what you would ask a professor in an email can be asked on Discord, and I would encourage you to ask it there. ",,,,,,, +8izBLk7thDN5rAWO,advanced_system_software_design_redacted,multi factual,What are the course objectives?,"The course objectives are as follows: 1. Demystify the seemingly complex interactions between software and hardware. 2. To familiarize with advanced issues in the design and implementation of modern software systems. 3. To understand how systems design principles apply to the design of computing systems, especially writing efficient and correct programs for multi-cores and multi-computers.","to demystify the seemingly complex interactions between +software and hardware",to familiarize with advanced issues in the design and implementation of modern software systems,"to understand how systems design principles apply to the +design of computing systems, especially writing efficient and correct programs for multi-cores and +multi-computers",,,,,,, +VNpRql9dXZLgqK8i,cs466_sp21_info_redacted,single reasoning,Can't I just review the slides online for the course for study?,"The slides are all available online but you will be tested on the modifications made to them in class, not the original slides.",,,,,,I will annotate slides in class. I will modify them somewhat to suit our needs. You are responsible for the modified slides as presented in class.,,,, +gAmEv8UzZHkfzRZU,ResEcon-213-Fall-2022,single factual,Where do I access homework?,In Connect.,"Each homework +will be posted in Connect.",,,,,,,,, +FSWXu02BPfxwqnC9,Most Recent P132 Syllabus - Spring 2022,summarization,What does the lab session cover in detail? ,Lab sessions meet once a week. It is not a reinforcement of lecture material but an independent opportunity for experimental data collection and data analysis. You work in pairs in lab sessions. There will be a practical exam covering lab sessions and that grade is added to your course grad at the end of the semester. ,"Unlike Physics 131, this course has a laboratory component that meets separately from the +other classroom sessions. The point of the lab is to provide an opportunity to develop data analysis skills using real data, NOT to reinforce content from lecture",The laboratory section has set of skills and goals that are separate and distinct from class,The lab is run quasi-independently from the rest of the course including an additional syllabus and Moodle page for the lab. ,"● The lab +does NOT meet the first week of class. +The lab meets every week the remainder of the semester - one week in the experimental area collecting data and one week in a classroom focusing on data analysis.","You will work in pairs during lab, but these groups will not be the same as your teams for class (too much of a logistical nightmare). +● There is a lab practical exam which you will complete on your own. +● Your lab grade will be folded in at the end of the semester",,,,, +6xtVkJ5lUYQ6AY78,ResEcon-213-Fall-2022,summarization,How do you set up your iClicker2 for classroom use?,"By September 19th, 2022, you must have either purchased or borrowed an iClicker2 transmitter and linked its ID with your Spire ID on Moodle.","You must link your +Spire ID to your iClicker2 transmitter on the Moodle site for this course. Note, there is a iClicker lending +program on campus so you can borrow one for the semester for free!","Each transmitter has a unique ID which you must +link to your SPIRE ID on Moodle. We begin using clickers on Monday, September 19, 2022.",,,,,,,, +502arQjor8mJytC0,ResEcon-213-Fall-2022,single reasoning,If I miss the midterm can I still succeed with this course?,Only if the absence is excused.,,,,,,"If you have an excused absence then the weight of the midterm will shift to the final exam, otherwise no.",,,, +qssMuKyao4MoOrZI,cs466_sp21_info_redacted,summarization,What are the academic dishonesty statement? ,"Academic dishonesty includes cheating, plagarism and fabrication. Those who saw any dishonesty can complain to the instructor."," Academic dishonesty is prohibited in all +programs of the University. ","Academic dishonesty includes but is not limited to: cheating, fabrication, +plagiarism, and facilitating dishonesty. ","Appropriate sanctions may be imposed on any student who +has committed an act of academic dishonesty. Instructors should take reasonable steps to address +academic misconduct. ","Any person who has reason to believe that a student has committed academic +dishonesty should bring such information to the attention of the appropriate course instructor as +soon as possible. ",,,,,, +j565KHahePQznlZr,Syllabus SPP 697 PM 2023 (clean),single factual,How can we contact during office hours?,Can contact via phone or zoom,"Contact Information and Office Hours + +Office Hours: On request, via phone or Zoom",,,,,,,,, +cO8ZhY8rmc9HHLG6,cs466_sp21_info_redacted,yes/no,Is this a graduate level course in cryptography?,No,Description: This is an undergraduate-level introduction to cryptography,,,,,,,,, +mUezDEQ7NgJVtyoP,ResEcon-213-Fall-2022,multi reasoning,Can I see the professor immediately with questions whenever I need to?,No.,,,,,,"The professor has scheduled hours, so probably not anytime you want",The syllabus says the teaching assistant is the primary contact for questions.,The syllabus lists a list of resources to use for questions and contacting the professor is last.,, +3EDZ55nZfFGOEy4W,Most Recent P132 Syllabus - Spring 2022,single reasoning,How important will computer skills be for this class,"You will need basic skills, and the ability to work with online programs. No advanced knowledge is needed, but you should no your way around a computer. ",,,,,,"Moderately as there are several portions of it, like Moodle, that need to be done online. ",,,, +qEQO1g2rjFAyPCWV,Syllabus SPP 697 PM 2023 (clean),multi reasoning,"Will this just be a pass fail course, or can I get rewarded for exceptional work/",This is not a pass or fail course. A grading scale of A through B- or below will be applied depending on the effort and quality of your work. ,,,,,,A Exceptional work with major effort and analysis sustained and material wellwritten and presented (95-100%),"A - Excellent work with few problems or questions, almost “A” (90-94%)","B+ Competent work with insights beyond simple understanding yet lacking in +effort/results to be considered “A” work (85-89%)","B Proficient understanding of basic material, acceptable level of work (81-84%)","B- or below Marginal level of work, demonstrated gaps in understanding or lack of +comprehension of the material at an acceptable level (80% or less)" +iq2E1RnP8d0FdiGS,ResEcon-213-Fall-2022,multi factual,What are the required materials for the class?,"You will need the Applied Statistics in Business & Economics textbook by David P. Doane, a personal response transmitter (an iClicker2), and computer access.","Required Materials: +1. Text: Applied Statistics in Business & Economics by David P. Doane and Lori E. Seward, 7th edition",Personal Response Transmitter (iClicker2),Computer Access,,,,,,, +NOG7IYGTbrzDgAQr,web_syllabus_4670,yes/no,Is this course available in the Fall Semester?,Yes,"Course Frequency: +Once per year (Fall semester)",,,,,,,,, +CKcDVC5xt9tPduP9,Most Recent P132 Syllabus - Spring 2022,multi reasoning,what is the structured of exams?,Two portions have each exam just like a quiz. That is Individual portion 75% of grade and colaborative portion 25%.,,,,,,"In Individual portion (75%) exam, participate in two different time slots","In Individual portion(75%)exam, attend the computer graded questions","In individual portion (75%) exam, bring your own device","In individual portion (75%) exam , moodle can be locked",the colaborative portion (25%) exam same as the individual exam that need all students and conducted on class room in next day +3qYJwadj2LUc8R3z,advanced_system_software_design_redacted,single reasoning,"I broke my arm last week, can I get help with note taking in class?",Yes. Register with disability services and get set up with their note-taking service as soon as possible. ,,,,,,A broken arm would be considered a disability. Disability services offers several accommodations for people once they're registered. ,,,, +35lkuiapH8NiDvPA,ResEcon-213-Fall-2022,multi reasoning,Can I get an B+ in the class without completing iClicker 2 Personal Responses or Connect Homeworks?,"No, you cannot get a B+ if you miss those two components as a B+ requires 83 and the maximum possible score you could get would be 81. ",,,,,,Personal Response - iClicker 2 7%,Connect Homeworks 12%,"Assuming a perfect score on all other components, the maximum class score is 81/100",A B+ requires a score of 83, +pa6BPlOP03a48qpR,ResEcon-213-Fall-2022,multi factual,What are the accepted pre-requisites for taking the course?,"The pre-requisites include: 1) Rec-Econ 212, or 2) Stat 240",Successful completion of Rec-Econ 212,"or Stat 240 are appropriate prerequisites. Please talk to me +if you are relying on any other coursework as a prerequisite.",,,,,,,, +SAYAPARGVwO90nDY,Syllabus SPP 697 PM 2023 (clean),single factual,What is the textbook used for a majority of the reading material used for the class?,"The majority of the reading material for the course will be from the textbook The Public Productivity and Performance Handbook, 3rd Edition, by Marc Holzer and Andrew Ballard, Eds. (2022).","No purchases are required, but a number of readings will draw from a new book: Marc Holzer +and Andrew Ballard, Eds. (2022). The Public Productivity and Performance Handbook, 3rd +Edition. New York, NY: Routledge Press.",,,,,,,,, +WSVWco7HtIMqDleZ,151 syllabus fall 2022 Rounds,yes/no,Will I be penalized for lateness if I complete a quiz at 7 PM Friday that was assigned earlier that week?,No,There will be quizzes every Wednesday (due by Friday at midnight). ,,,,,,,,, +wmVc1BYAmCLANHdx,ResEcon-213-Fall-2022,single factual,"What is the downloadable, or E-book reading material requirement for the course?",The online access tool called connect.,"An +online tool called Connect comes with e-textbook access. Connect is an integral part of this course, and +everyone must have it",,,,,,,,, +bIPYLM0YWqUhyxUS,Most Recent P132 Syllabus - Spring 2022,multi reasoning,"The learning is intrinsically active, what will happen if only lecture had happened?",It had become passive process and cannot become to put into their heads.,,,,,,The lecture is a passive process which cannot put information in the heads.,We need to spend time to solve problems.,Need to apply ideas to new situations.,Need to connect information to the stuff., +FsyqeZfxC8GxbZMz,151 syllabus fall 2022 Rounds,summarization,What can I use Moodle for and how frequently should I check it if its a course requirement? ,"Moodle should be used frequently and gives you access to all readings, homework, lectures, quizzes and grades. ", required reading,homework questions,pre- lecture notes,post-lecture notes,recordings of every lecture,,,,, +avA3T2SAdyrK142Q,cs466_sp21_info_redacted,single factual,How much homework assignments does each will be assigned?,Six homework assignment,There will be six homework assignments worth 100 points each.,,,,,,,,, +d5mOzlsiOiHc9KPN,Syllabus_Marine_Microbiome_2022_redacted,summarization,Could you explain the course description?,The course involves microbial being examined and viruses in marine enviroments. ,"Course Description: +This course studies the total of microorganisms and viruses in marine environments +using theories. ","The latest technologies and methods used in marine microbial ecology +will be examined.",,,,,,,, +z4xZUxWLKAjGGA8k,151 syllabus fall 2022 Rounds,multi reasoning,"If I don't do very well on one of my midterms, will I be able to pull up my grade somehow?","If you get a higher grade on your final exam than on your midterm, the midterm grade will be dropped giving you a higher overall grade for the course.",,,,,,"The syllabus states if you get a higher grade on your final exam then you do on your midterm, the instructor will drop your midterm exam grade.","So if you do better on your final exam, your lowest midterm grade is dropped which will increase your final score in the course.",,, +BD9Xc40TTMt8NK6s,151 syllabus fall 2022 Rounds,single reasoning,"I will not be able to attend on the first midterm exam, will this impact my grade?","Being present and doing the midterms will help on your overall understanding of the material. It is suggested you take them, however missing them will not impact your final score.",,,,,,The midterms in this course are not cummulative.,,,, +drPCoF2kLn2PjGF6,advanced_system_software_design_redacted,summarization,What does the professor say about their policy on personal pronouns and names? ,The professor says that everyone has the right to be addressed by the name and pronoun of their choosing and that they can enter those on SPIRE. The professor will also give the opportunity to let them know if those requests are not on the class roster. The professor also wants people to know that their choices will be respected in class at all times. ,Everyone has the right to be addressed by the name and pronouns that they use for themselves.,"Students can indicate their preferred/chosen first name and pronouns on SPIRE, which appear on class rosters.",Please let me know what name and pronouns I should use for you if they are not on the roster.,A student’s chosen name and pronouns are to be respected at all times in the classroom.,,,,,, +nfSATy6pFAXiHCMo,151 syllabus fall 2022 Rounds,yes/no,Does this course require BlackBoard? ,No,"Required Materials: Moodle, iClicker Cloud, E-mail, and Discord. ",,,,,,,,, +cikUw2negqesSoxh,151 syllabus fall 2022 Rounds,summarization,Can you tell me about how much reading I will have to do for the class?,"There is no textbook for this class, however, there will be ungraded reading assignment throughout the course. Will the reading is not required, the information may be important to know for your exams.",There is no required textbook for this class. ,There will be reading/viewing each week there is no grade attached to these assignments. There will be an ungraded reading guide posted. This information will help you with exams.,"Because this is ungraded, you have a choice whether to do it or not. ","However, I will not necessarily directly cover all of the concepts in the reading guide in class and you are responsible for them. ",,,,,, +Wr7JRF56U6oaYcqZ,advanced_system_software_design_redacted,yes/no,Can I refer text book in this course?,Yes,"Recommended Text Books +OSTEP",,,,,,,,, +6nPmFQCFxJ2WKpSg,Syllabus_Marine_Microbiome_2022_redacted,multi factual,How will the final grade for this class be determined?,"The final grade will consist of a balance of 50% course project, 20% participation in class, and 30% from oral presentations. There is also a grading scheme where anything from 90-100 is and A, 89-80 is an B, 79-70 is a C, and 67-69 is a D.","Final grades will be calculated using the following weighting: +",50% Course project,20% Participation in class discussions,30% Oral presentations,"Grading scheme: +(90-100=A; 87-89=B+; 80-86=B, 77-79=C+; 70-76=C; 67-69=D+; 60-66=D; <60=F)",,,,, +9QRfLN4LZZouD4tN,Syllabus_Marine_Microbiome_2022_redacted,single reasoning,What do I do if I am unable to attend class? ,Bring a doctor's note or let the professor know beforehand. ,,,,,,Missing class is ok if you are sick or have it scheduled. ,,,, +SGR9tdvMtWVL05vo,web_syllabus_4670,multi reasoning,How many assignments will I have to complete in the course?,"There will be seven weekly homeworks, two exams, and five design projects for a total of 14 assignments over the course.",,,,,,Homework: weekly assignments. Approximately seven homework assignments per semester.,Exams: Two preliminary exams.,"Design projects: There are five labs, which form the centerpiece for the course.",, +nSkfpPZnNQp0XC4d,151 syllabus fall 2022 Rounds,single reasoning,"If I do poorly on one of the exams, what is the best way to improve my final grade?","By scoring higher on your final exam than you did on your mid-term. In this case, the mid-term grade will be replaced by the final exam grade.",,,,,,"The teacher will replace the lowest mid-term score with the final exam score, if the final exam score is higher.",,,, +9VCdsFGIoi9ciIt8,cs466_sp21_info_redacted,single factual,How many points are allotted for each homework assignment?,"There will be six homework assignments, each worth 100 points.",Points: 100/homework assignment,,,,,,,,, +yqQnSUGTZ1Q0B9W6,web_syllabus_4670,yes/no,Will I need text books or other preparation materials?,No,None. Lecture notes will be provided.,,,,,,,,, +D1sitkTEyLZA1Z6l,cs466_sp21_info_redacted,single factual,How many homework assignments will there be?,6,There will be six homework assignments worth 100 points each.,,,,,,,,, +3qyuC3vo9SzEUpS8,cs466_sp21_info_redacted,single factual,How many units is this course worth?,Three units.,This is an undergraduate-level introduction to cryptography,,,,,,,,, +KcUU2OyjrBVkPb7P,Syllabus_Marine_Microbiome_2022_redacted,multi factual,What are the requirements for the oral presentations students must give during this course?,"Each student will ultimately deliver three oral presentations to the class based on assigned researcher papers, with the intention of teaching the material to the other students in the class.",Each student will give three in-class oral presentations on assigned topics.,Each student will give three 30-min oral presentation to the class on an assigned paper followed by discussions. ,"Presentations should focus on teaching the material to the +class.",,,,,,, +Z7XhFNVQQMVz5dXE,Syllabus_Marine_Microbiome_2022_redacted,single reasoning,Is it a morning class?,No,,,,,,"The time says PM, and so it must be in the afternoon or evening.",,,, +Gci9GSICRJUyboAG,cs466_sp21_info_redacted,single factual,What is the cutoff to receive a grade of A in this class using the raw score chart?,The cutoff for a grade of A based on the raw score is between 0.8 and 1.,"Your grade will be no lower than the following cutoffs on the raw score: +.8 to 1 A +.6 to .799 B +.4 to .599 C +< .4 F",,,,,,,,, +ESkkF4gkOpoqGXfq,ResEcon-213-Fall-2022,summarization,What aspects make up the bulk of the student's grade in this course?,"65% of the total grade is based on exams, with the final exam being weighted the most at 30%. This is more than half the total. Quizzers and homework make up the bulk of remaining grade, at 13% and 12% respectively. Attendance, discussion, and personal responses are negligible amounts.",Course scores are calculated based on weightings.,Final exam is worth 30%.,The two midterms are worth 35% total.,Quizzes are worth 13%.,Homework is worth 12%.,,,,, +hHzLrEFgAsOOvDgO,cs466_sp21_info_redacted,yes/no,Does the professor have a physical office I can meet them in?,No,Office: Zoom,,,,,,,,, +pahwzDsH6zh5z2hK,Most Recent P132 Syllabus - Spring 2022,multi reasoning,Should homework assignments be done in one sitting?,The long homework assignments are meant to be done at a steady pace over a period of time.,,,,,,The assignments are very long. ,It is recommended the students answers the questions as they read the text over several assignments. ,"If the student runs into a problems, they are recommended to reach out for help.",, +PuMFTHvTE1TOLZIj,Syllabus SPP 697 PM 2023 (clean),summarization,Will there be any breaks during the semister,There will be on class on March 14 or April 18.,"Course Outline +Session 1 – Course Orientation, Introductions, and Overview of the Evolution of the +Performance Management Movement in the U.S. (February 7) +Session 2 - Creating a Performance Context with Strategic Outcomes, Strategic Foresight, +Strategic Planning, and Enterprise Risk Management (February 14) +Session 3 - Alternative Approaches and Tools for Creating Performance Management +Frameworks (February 21) +Session 4 - Selecting, Defining, and Collecting Data and Performance Measures (February 28) +Session 5 - Analyzing and Using Data and Evidence to Inform Decisions (March 7) +-- Spring Break; No Class on March 14 --","Session 6 - Analyzing and Using Data and Evidence to Improve Performance (March 21) +Session 7 - Creating Incentives and Accountability for Performance (March 28) +Session 8 – Using Evidence and Evaluation Techniques (April 4) +Session 9 – Building Institutional Capacity to Ensure Sustainability (April 11) +-- Patriots’ Day Holiday; No Class on April 18 --",,,,,,,, +0GOsCz80uAL2yvWk,Syllabus_Marine_Microbiome_2022_redacted,multi factual,How should students prepare for each class when they aren't giving a presentation?,"In order to adequately prepare for each class, students are expected to do the reading prior to class and submit at least one question regarding the topic to the instructor. When in class, students should be ready to participate and learn.",Non-presenting students are required to read the articles before the student presentation and submit at least one question to the instructor before class. ,come to class prepared to contribute and learn,do assigned readings before class and take notes during class,,,,,,, +DY8bnsRXS4XTlzZf,151 syllabus fall 2022 Rounds,multi factual,How Many Exams do we have in this course?,"(2) Exams, (1) Final Exam, (2)None Cumulative Midterms. Quizzes on Wednesdays.",Two examinations will be held on 6 October and 10 November,"A final exam +will be held during finals period the date, time and room will be on spire","The two mid-term exams are not cumulative – you will be assessed on the material covered since the previous +exam",Quizzes will be based on the lectures and homework in the week the quizzes are given.,,,,,, +BiCRaIDkb02xe5Nc,ResEcon-213-Fall-2022,yes/no,Is there a required textbook?,Yes,"1. Text: Applied Statistics in Business & Economics by David P. Doane and Lori E. Seward, 7th edition.",,,,,,,,, +rSsAXGPDQsf49Btz,ResEcon-213-Fall-2022,multi factual,"What are the course requirements, in terms of testing for the class?","The tests required for the course are as follows: 1) Midterm 1, 2) Midterm 2, 3) Final exam.",Examinations: There are 2 mid-semester examinations and a final exam.,"Midterm 1 TBD 10/19/2022 7:00-9:00pm +","Midterm 2 TBD 11/16/2022 7:00-9:00pm +",,,,,,, +4JUC3ynx8NgOEH6X,web_syllabus_4670,summarization,Can you provide an overview of what the labs will entail?,"There are a total of five labs. The first two labs are focused on learning the hardware setups used in class. In the third lab, you will design a communication system for a simple channel. The fourth lab, you will design a basic communication system for actual hardware. The fifth lab is the most complex and you will design two separate communication systems, one optimized for performance and one adhering to standards.",The first two labs are designed to acclimate students with the hardware setup that we will be using. , The third lab involves designing a communication system for a simple channel simulated in software.,"The fourth lab involves designing a basic communication system for the actual, hardware-based channel and +submitting a related “standard” that one could use for designing compatible transmitters and receivers +for this channel. ","In the fifth lab, students will design two communication systems, one that complies with the standard and one that +aims for the highest performance. ",,,,,, +zxeFZQMkhcbOXuKW,Most Recent P132 Syllabus - Spring 2022,multi factual,Which day and the time does the class meets?,"Class meets on monday, wednesday and friday. The time for both section varies section 1 held on 10.10 am-11.00 am and section 2 on 9.05 and 9.55 am.","Class meets Monday, Wednesdays and Fridays +","Times Section 2: 9:05 – 9:55am +Section 1: 10:10 – 11:00 am",,,,,,,, +cEQ2IOULszy9CtKP,Syllabus_Marine_Microbiome_2022_redacted,single reasoning,I work all day Monday. Can I still take the class?,Yes you may. The class schedule will not conflict with your work schedule.,,,,,,Schedule: Tuesday & Thursday ,,,, +6cccSS7NwBe9ABE3,web_syllabus_4670,single reasoning,"If I don't bother handing in homework, can I still pass the class?","Homework is 15% of your total grade, so if you theoretically get a perfect score on your prelims and labs then you can hope for an 85% in the class and will pass, yes.",,,,,,"Course Grading Scheme: 15% homework assignments, 50% labs, 35% prelims",,,, +XidjUBuW2okuW5yL,cs466_sp21_info_redacted,multi factual,What is the makeup policy and what if I have an excused absence? ,"Makeups will not be accepted on homework. Makeups on the exams will be given at the discretion of the instructor, however the absence must be legitimate and verifiable. If the absence is excused the makeup exam will be given within one week of the missed test. If homework is missed, the lowest homework score will be dropped.",There will be no makeups on homework.,"Makeups on an exam will be given at the discretion of the +instructor. ","To allow for excused absences, I +will drop your lowest homework score.", A legitimate and verifiable excuse is required.,"If the excuse is approved, the makeup will +be given within one week of the missed test.",,,,, diff --git a/benchmark/RAG/run.py b/benchmark/RAG/run.py new file mode 100644 index 00000000..ee30a3dc --- /dev/null +++ b/benchmark/RAG/run.py @@ -0,0 +1,169 @@ +import os +import sys +import yaml +import importlib +from argparse import ArgumentParser +from pathlib import Path + +sys.path.append(str(Path(__file__).parent)) + +from src.core.logger import setup_logging +# ========================================== +# 1. Environment Initialization +# ========================================== +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +PROJECT_ROOT = SCRIPT_DIR + +ov_config_path = os.path.join(SCRIPT_DIR, "ov.conf") +if os.path.exists(ov_config_path): + os.environ["OPENVIKING_CONFIG_FILE"] = ov_config_path + print(f"[Init] Auto-detected OpenViking config: {ov_config_path}") + +try: + from src.pipeline import BenchmarkPipeline + from src.core.vector_store import VikingStoreWrapper + from src.core.llm_client import LLMClientWrapper +except SyntaxError as e: + print(f"\n[Fatal Error] Syntax error while importing modules: {e}") + sys.exit(1) +except ImportError as e: + print(f"\n[Fatal Error] Cannot import modules: {e}") + print(f"Current sys.path: {sys.path}\n") + sys.exit(1) + +# ========================================== +# 2. Helper Functions +# ========================================== + +def load_config(config_path): + if not os.path.exists(config_path): + raise FileNotFoundError(f"Config file not found: {config_path}") + with open(config_path, 'r', encoding='utf-8') as f: + return yaml.safe_load(f) + +def resolve_path(path_str, base_path): + """ + Convert relative path to absolute path based on base_path. + If path_str is already absolute, keep it unchanged. + """ + if not path_str: + return path_str + if os.path.isabs(path_str): + return path_str + return os.path.normpath(os.path.join(base_path, path_str)) + +# ========================================== +# 3. Main Program +# ========================================== + +def main(): + parser = ArgumentParser(description="Run RAG Benchmark (Smart Path Handling)") + default_config_path = os.path.join(SCRIPT_DIR, "config/config.yaml") + + parser.add_argument("--config", default=default_config_path, + help=f"Path to config file. Default: {default_config_path}") + + parser.add_argument("--step", choices=["all", "gen", "eval", "del"], default="all", + help="Execution step: 'gen' (Retrieval+LLM), 'eval' (Judge), or 'all'") + + args = parser.parse_args() + + # --- B. Load and Parse Config --- + config_path = os.path.abspath(args.config) + print(f"[Init] Loading configuration from: {config_path}") + + try: + config = load_config(config_path) + except FileNotFoundError as e: + print(f"[Error] {e}") + return + + # --- C. Path Resolution --- + print(f"[Init] Resolving paths relative to Project Root: {PROJECT_ROOT}") + dataset_name = config.get('dataset_name', 'UnknownDataset') + retrieval_topk = config.get('execution', {}).get('retrieval_topk', 5) + + format_vars = { + 'dataset_name': dataset_name, + 'retrieval_topk': retrieval_topk + } + + path_keys = ['raw_data', 'output_dir', 'vector_store', 'log_file', 'doc_output_dir'] + for key in path_keys: + if key in config.get('paths', {}): + original = config['paths'][key] + rendered_path = original.format(**format_vars) + resolved = resolve_path(rendered_path, PROJECT_ROOT) + config['paths'][key] = resolved + # print(f" - {key}: {resolved}") + + # --- D. Initialize Components --- + try: + logger = setup_logging(config['paths']['log_file']) + logger.info(">>> Benchmark Session Started") + + # 1. Adapter (Dynamic Loading) + adapter_cfg = config.get('adapter', {}) + module_path = adapter_cfg.get('module', 'src.adapters.locomo_adapter') + class_name = adapter_cfg.get('class_name', 'LocomoAdapter') + + logger.info(f"Dynamically loading Adapter: {class_name} from {module_path}") + logger.info(f"Loading raw data from: {config['paths']['raw_data']}") + + try: + mod = importlib.import_module(module_path) + AdapterClass = getattr(mod, class_name) + adapter = AdapterClass(raw_file_path=config['paths']['raw_data']) + except ImportError as e: + logger.error(f"Could not import module '{module_path}'. Please check your config 'adapter.module'. Error: {e}") + raise e + except AttributeError as e: + logger.error(f"Class '{class_name}' not found in module '{module_path}'. Please check your config 'adapter.class_name'. Error: {e}") + raise e + + # 2. Vector Store + vector_store = VikingStoreWrapper(store_path=config['paths']['vector_store']) + + # 3. LLM Client + api_key = os.environ.get( + config['llm'].get('api_key_env_var', ''), + config['llm'].get('api_key') + ) + if not api_key: + logger.warning("No API Key found in config or environment variables!") + + llm_client = LLMClientWrapper(config=config['llm'], api_key=api_key) + + # 4. Pipeline + pipeline = BenchmarkPipeline( + config=config, + adapter=adapter, + vector_db=vector_store, + llm=llm_client + ) + + # --- E. Execute Tasks --- + if args.step in ["all", "gen"]: + logger.info("Stage: Generation (Ingest -> Retrieve -> Generate)") + pipeline.run_generation() + + if args.step in ["all", "eval"]: + logger.info("Stage: Evaluation (Judge -> Metrics)") + pipeline.run_evaluation() + + if args.step in ["all", "del"]: + logger.info("Stage: Delete Vector Store") + pipeline.run_deletion() + + logger.info("Benchmark finished successfully.") + + except KeyboardInterrupt: + print("\n[Stop] Execution interrupted by user.") + except Exception as e: + if 'logger' in locals(): + logger.exception("Fatal error during execution") + print(f"\n[Fatal Error] Program execution error: {str(e)}") + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/benchmark/RAG/src/__init__.py b/benchmark/RAG/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/benchmark/RAG/src/adapters/__init__.py b/benchmark/RAG/src/adapters/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/benchmark/RAG/src/adapters/base.py b/benchmark/RAG/src/adapters/base.py new file mode 100644 index 00000000..954b2227 --- /dev/null +++ b/benchmark/RAG/src/adapters/base.py @@ -0,0 +1,79 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import List, Dict, Any, Union, Optional +import sys +from pathlib import Path + +sys.path.append(str(Path(__file__).parent.parent)) + +from core.logger import get_logger + + +@dataclass +class StandardQA: + """Standardized single question-answer pair""" + question: str + gold_answers: List[str] + evidence: List[str] = field(default_factory=list) + category: Optional[Union[int, str]] = None + metadata: Dict[str, Any] = field(default_factory=dict) + + +@dataclass +class StandardSample: + """Standardized sample containing document content and corresponding QA list""" + sample_id: str + qa_pairs: List[StandardQA] + metadata: Dict[str, Any] = field(default_factory=dict) + + +@dataclass +class StandardDoc: + """Standardized sampleid to doc_path mapping structure""" + sample_id:str + doc_path:str + + +class BaseAdapter(ABC): + """Base class for all dataset adapters""" + + def __init__(self, raw_file_path: str): + self.raw_file_path = raw_file_path + self.logger = get_logger() + + @abstractmethod + def data_prepare(self, doc_dir:str) -> List[StandardDoc]: + """ + Data preparation. + 1. Convert dataset format to OpenViking-friendly format + 2. Return converted (or unconverted) file paths + + Returns: + List[StandardDoc]: Array of file paths expected to be input to OpenViking + """ + pass + + @abstractmethod + def load_and_transform(self) -> List[StandardSample]: + """ + Read raw files and convert to standard format list. + Must be implemented by subclasses. + """ + pass + + @abstractmethod + def build_prompt(self, qa: StandardQA, context_blocks: List[str]) -> tuple[str, Dict[str, Any]]: + """ + Build final prompt to send to LLM based on retrieved context and QA pair. + + Returns: + - full_prompt (str): Complete prompt string + - meta (Dict): Metadata to pass to post-processing function (e.g., option mapping for multiple choice) + """ + pass + + def post_process_answer(self, qa: StandardQA, raw_answer: str, meta: Dict[str, Any]) -> str: + """ + Post-process raw LLM output (default implementation only strips whitespace). + """ + return raw_answer.strip() diff --git a/benchmark/RAG/src/adapters/financebench_adapter.py b/benchmark/RAG/src/adapters/financebench_adapter.py new file mode 100644 index 00000000..121b66f1 --- /dev/null +++ b/benchmark/RAG/src/adapters/financebench_adapter.py @@ -0,0 +1,161 @@ +# src/adapters/finance_bench_adapter.py +""" +FinanceBench Dataset Adapter + +FinanceBench is a financial domain QA dataset with SEC financial report PDFs as documents. +Data format: JSONL, each line contains question, answer, doc_name, evidence, etc. +evidence_text in evidence is used for recall calculation. +""" + +import json +import os +from collections import defaultdict +from typing import List, Dict, Any +from pathlib import Path +import sys + +sys.path.append(str(Path(__file__).parent)) + +from base import BaseAdapter, StandardDoc, StandardSample, StandardQA + +CATEGORY_INSTRUCTIONS = { + "domain-relevant": """Answer the financial question based on the document. +- Use ONLY facts from the context +- If numerical, include units (e.g., USD millions, %) +- Provide concise, direct answer +- Do NOT invent information""", + + "metrics-generated": """Calculate the financial metric based on the document. +- Use ONLY numbers from the context +- Show your calculations clearly +- Round to appropriate decimal places +- Include units (e.g., USD millions, %) +- Do NOT invent numbers""", + + "novel-generated": """Answer the financial question based on the document. +- Use ONLY facts from the context +- If numerical, include units (e.g., USD millions, %) +- Provide clear, complete answer +- Do NOT invent information""" +} + +MISSING_RULE = "If the provided context does not contain sufficient information to answer the question, respond with 'Insufficient information'." + + +class FinanceBenchAdapter(BaseAdapter): + """ + FinanceBench Dataset Adapter. + Processes financial domain QA data with SEC financial report PDFs as documents. + """ + + def __init__(self, raw_file_path: str): + super().__init__(raw_file_path) + data_dir = os.path.dirname(self.raw_file_path) + self.pdf_dir = os.path.join(data_dir, "pdfs") + + def data_prepare(self, doc_dir: str) -> List[StandardDoc]: + """ + Prepare document list for ingestion. Only ingest documents referenced in JSONL. + """ + if not os.path.exists(self.pdf_dir): + raise FileNotFoundError(f"PDF directory not found: {self.pdf_dir}") + + doc_names = set() + with open(self.raw_file_path, 'r', encoding='utf-8') as f: + for line in f: + line = line.strip() + if line: + doc_names.add(json.loads(line)["doc_name"]) + + docs: List[StandardDoc] = [] + for doc_name in sorted(doc_names): + pdf_path = os.path.join(self.pdf_dir, f"{doc_name}.pdf") + if not os.path.exists(pdf_path): + self.logger.warning(f"PDF not found: {pdf_path}, skipping") + continue + docs.append(StandardDoc(sample_id=doc_name, doc_path=pdf_path)) + + self.logger.info(f"[FinanceBench] Prepared {len(docs)} documents for ingestion (referenced only)") + return docs + + def load_and_transform(self) -> List[StandardSample]: + """ + Parse JSONL question file, group by doc_name into StandardSample. + evidence uses evidence_text field from each evidence. + """ + if not os.path.exists(self.raw_file_path): + raise FileNotFoundError(f"Raw data file not found: {self.raw_file_path}") + + groups: Dict[str, List[Dict]] = defaultdict(list) + with open(self.raw_file_path, 'r', encoding='utf-8') as f: + for line in f: + line = line.strip() + if not line: + continue + item = json.loads(line) + groups[item["doc_name"]].append(item) + + samples: List[StandardSample] = [] + for doc_name, items in groups.items(): + qa_pairs = [] + for item in items: + evidence_texts = [ + ev["evidence_text"] + for ev in item.get("evidence", []) + if ev.get("evidence_text") + ] + + qa_pairs.append(StandardQA( + question=item["question"], + gold_answers=[item["answer"]], + evidence=evidence_texts, + category=item.get("question_type"), + metadata={ + "financebench_id": item.get("financebench_id"), + "question_reasoning": item.get("question_reasoning"), + "justification": item.get("justification", ""), + "company": item.get("company"), + } + )) + + samples.append(StandardSample( + sample_id=doc_name, + qa_pairs=qa_pairs, + )) + + self.logger.info( + f"[FinanceBench] Loaded {sum(len(s.qa_pairs) for s in samples)} questions " + f"across {len(samples)} documents" + ) + return samples + + def build_prompt(self, qa: StandardQA, context_blocks: List[str]) -> tuple[str, Dict[str, Any]]: + context_text = "\n\n".join(context_blocks) + + category = qa.category + category_instruction = CATEGORY_INSTRUCTIONS.get(category, "") + + if category_instruction: + full_prompt = f"""{context_text} + +{category_instruction} + +{MISSING_RULE} + +Question: {qa.question} + +Answer:""" + else: + full_prompt = f"""{context_text} + +{MISSING_RULE} + +Question: {qa.question} + +Answer:""" + + meta = { + "question_type": qa.category, + "financebench_id": qa.metadata.get("financebench_id"), + } + return full_prompt, meta diff --git a/benchmark/RAG/src/adapters/locomo_adapter.py b/benchmark/RAG/src/adapters/locomo_adapter.py new file mode 100644 index 00000000..81291b8f --- /dev/null +++ b/benchmark/RAG/src/adapters/locomo_adapter.py @@ -0,0 +1,180 @@ +# src/adapters/locomo_adapter.py +import json +import os +from typing import List, Dict, Any + +from .base import BaseAdapter, StandardDoc, StandardSample, StandardQA + + +MISSING_RULE = "If no information is available to answer the question, write 'Not mentioned'." + + +CATEGORY_INSTRUCTIONS = { + "1": """Extract the exact factual answer from the conversation. +- Use the exact words from the context when possible +- If multiple items, separate with commas""", + + "2": """Answer the time-related question. +- Pay close attention to DATE labels in the conversation +- Calculate relative time (e.g., "10 years ago") when needed +- Use the exact dates from the context""", + + "3": """Reason and infer based on the conversation. +- Use ONLY the facts in the context +- State your conclusion clearly (e.g., "Likely yes", "Probably no") +- Do NOT explain your reasoning or provide any basis/justification +- Only output your final conclusion, nothing else +- Do NOT invent information""", + + "4": """Understand the meaning and significance. +- Focus on what the speakers mean, not just what they say +- Identify symbolism or implied meaning +- Use wording from the context when possible""", +} + + +class LocomoAdapter(BaseAdapter): + """ + Adapter specifically for processing the LocoMo dataset. + Converts session-format JSON to Markdown with time information. + """ + def data_prepare(self,doc_dir:str) -> List[StandardDoc]: + """ + Load raw data and convert to OpenViking-friendly format + """ + if not os.path.exists(self.raw_file_path): + raise FileNotFoundError(f"Raw data file not found: {self.raw_file_path}") + + res:List[StandardDoc] = [] + + with open(self.raw_file_path, 'r', encoding='utf-8') as f: + data = json.load(f) + dataset = [data] if isinstance(data, dict) else data + os.makedirs(doc_dir, exist_ok=True) + for item in dataset: + sample_id = item.get("sample_id", "unknown") + doc_content = self._convert_conversation_to_markdown(sample_id, item.get("conversation", {})) + + try: + doc_path = os.path.join(doc_dir, f"{sample_id}_doc.md") + with open(doc_path, "w", encoding="utf-8") as f: + f.write(doc_content) + res.append(StandardDoc(sample_id, doc_path)) + except Exception as e: + self.logger.error(f"[locomo adapter] doc:{sample_id} prepare error {e}") + raise e + return res + + def load_and_transform(self) -> List[StandardSample]: + """ + Load raw JSON data and convert to standardized StandardSample object list. + """ + if not os.path.exists(self.raw_file_path): + raise FileNotFoundError(f"Raw data file not found: {self.raw_file_path}") + + with open(self.raw_file_path, 'r', encoding='utf-8') as f: + data = json.load(f) + dataset = [data] if isinstance(data, dict) else data + + standard_samples = [] + + for item in dataset: + sample_id = item.get("sample_id", "unknown") + + qa_pairs = [] + for q in item.get("qa", []): + if str(q.get("category")) == "5": + continue + raw_ans = q.get("answer") + + if isinstance(raw_ans, list): + golds = raw_ans + elif raw_ans is None or raw_ans == "": + golds = ["Not mentioned"] + else: + golds = [raw_ans] + + qa_pairs.append(StandardQA( + question=q["question"], + gold_answers=[str(g) for g in golds], + evidence=q.get("evidence", []), + category=q.get("category"), + metadata={"original_id": q.get("id")} + )) + + standard_samples.append(StandardSample( + sample_id=sample_id, + qa_pairs=qa_pairs + )) + + return standard_samples + + def _convert_conversation_to_markdown(self, sample_id: str, conv: Dict[str, Any]) -> str: + """ + Convert LocoMo session structure to flat Markdown string. + """ + md_lines = [f"# Chat History: {sample_id}"] + + session_idx = 1 + while f"session_{session_idx}" in conv: + s_key = f"session_{session_idx}" + dt_key = f"session_{session_idx}_date_time" + sum_key = f"session_{session_idx}_summary" + + md_lines.append(f"\n## Session {session_idx}") + + session_dt = conv.get(dt_key) + if session_dt: + md_lines.append(f"DATE: {session_dt}") + + session_sum = conv.get(sum_key) + if session_sum: + md_lines.append(f"SUMMARY: {session_sum}") + + for turn in conv[s_key]: + spk = turn.get("speaker", "Unknown") + txt = turn.get("text", "") + + raw_id = turn.get("dia_id") or turn.get("id") + suffix = f" [{raw_id}]" if raw_id else "" + + md_lines.append(f"**{spk}**: {txt}{suffix}") + + session_idx += 1 + + return "\n".join(md_lines) + + def build_prompt(self, qa: StandardQA, context_blocks: List[str]) -> tuple[str, Dict[str, Any]]: + category = str(qa.category) + context_text = "\n\n".join(context_blocks) + + category_instruction = CATEGORY_INSTRUCTIONS.get(category, "") + + if category_instruction: + full_prompt = f"""{context_text} + +{MISSING_RULE} + +--- +{category_instruction} + +Question: {qa.question} + +Answer:""" + else: + full_prompt = f"""{context_text} + +{MISSING_RULE} + +Based on the conversation above, answer the following question. +Use ONLY the provided context. Do NOT invent any information. + +Question: {qa.question} + +Answer:""" + + meta = {"category": category} + return full_prompt, meta + + def post_process_answer(self, qa: StandardQA, raw_answer: str, meta: Dict[str, Any]) -> str: + return raw_answer.strip() diff --git a/benchmark/RAG/src/adapters/qasper_adapter.py b/benchmark/RAG/src/adapters/qasper_adapter.py new file mode 100644 index 00000000..1454e6ac --- /dev/null +++ b/benchmark/RAG/src/adapters/qasper_adapter.py @@ -0,0 +1,400 @@ +# src/adapters/qasper_adapter.py +""" +Qasper Dataset Adapter + +Qasper is an academic paper QA dataset containing 1585 NLP papers and 5049 questions. +Each question is answered by multiple annotators, with answer types including: +- extractive_spans: text spans extracted from the paper +- free_form_answer: free-form answers +- yes_no: yes/no answers +- unanswerable: no answer can be found in the paper + +Dataset characteristics: +1. Each paper includes title, abstract, section content, and figure/table information +2. Each question may have answers from multiple annotators +3. Each answer has corresponding evidence (evidence text) + +Adapter functions: +- data_prepare: Convert papers to Markdown format, preserving section structure +- load_and_transform: Parse QA data, preserving answer-evidence correspondence +- build_prompt: Build QA prompt +- post_process_answer: Post-process LLM output +""" + +import json +import os +from typing import List, Dict, Any + +from .base import BaseAdapter, StandardDoc, StandardSample, StandardQA + +# Specific instructions for different answer types +CATEGORY_INSTRUCTIONS = { + "extractive": """Extract the exact answer from the paper. +- Use EXACT wording from the context +- Do NOT rephrase or add explanation +- Provide concise, direct answer""", + + "free_form": """Answer using information from the paper. +- Use ONLY facts from the context +- You may rephrase or summarize in your own words +- Provide clear, complete answer +- Do NOT invent information""", + + "yes_no": """Answer Yes/No question based on the paper. +- First respond "Yes" or "No" +- Do NOT add explanation +- Use ONLY info from context +- Do NOT invent information""" +} + +# Rule for when answer cannot be found +MISSING_RULE = "If no information is available to answer the question, write 'Not mentioned'." + + +class QasperAdapter(BaseAdapter): + """ + Adapter specifically for processing Qasper dataset. + + Converts academic papers to Markdown documents with section structure, + and converts QA data to standardized StandardSample format. + + Attributes: + raw_file_path: Raw JSON data file path + logger: Logger + """ + + def data_prepare(self, doc_dir: str) -> List[StandardDoc]: + """ + Load raw data and convert to OpenViking-friendly format. + + Converts each paper to Markdown document, preserving: + - Title (# Title) + - Abstract (## Abstract) + - Sections (## Section Name) + - Figures and Tables (## Figures and Tables) + + Args: + doc_dir: Document output directory path + + Returns: + List[StandardDoc]: List of standardized document objects, each containing paper_id and document path + + Raises: + FileNotFoundError: Raw data file not found + """ + if not os.path.exists(self.raw_file_path): + raise FileNotFoundError(f"Raw data file not found: {self.raw_file_path}") + + res: List[StandardDoc] = [] + + with open(self.raw_file_path, 'r', encoding='utf-8') as f: + data = json.load(f) + + os.makedirs(doc_dir, exist_ok=True) + + for paper_id, paper_data in data.items(): + doc_content = self._convert_paper_to_markdown(paper_id, paper_data) + + try: + doc_path = os.path.join(doc_dir, f"{paper_id}_doc.md") + with open(doc_path, "w", encoding="utf-8") as f: + f.write(doc_content) + res.append(StandardDoc(paper_id, doc_path)) + except Exception as e: + self.logger.error(f"[qasper adapter] doc:{paper_id} prepare error {e}") + raise e + return res + + def load_and_transform(self) -> List[StandardSample]: + """ + Load raw JSON data and convert to standardized StandardSample object list. + + Processing logic: + 1. Iterate through QA list of each paper + 2. For each question, collect answers from all annotators + 3. Preserve answer-evidence correspondence (stored in metadata) + 4. Format question as "Based on the paper \"{title}\", {question}" + + Answer type handling: + - extractive_spans: directly use extracted text spans + - free_form_answer: use free-form answer + - yes_no: convert to "Yes" or "No" + - unanswerable: convert to "Not mentioned" + + Returns: + List[StandardSample]: List of standardized sample objects + + Raises: + FileNotFoundError: Raw data file not found + """ + if not os.path.exists(self.raw_file_path): + raise FileNotFoundError(f"Raw data file not found: {self.raw_file_path}") + + with open(self.raw_file_path, 'r', encoding='utf-8') as f: + data = json.load(f) + + standard_samples = [] + + for paper_id, paper_data in data.items(): + qa_pairs = [] + paper_title = paper_data.get("title", "Unknown Title") + + for qa_item in paper_data.get("qas", []): + + # --- Unanswerable filtering logic --- + # Check if all answers are marked as unanswerable + is_unanswerable = all( + ans.get("answer", {}).get("unanswerable", False) + for ans in qa_item.get("answers", []) + ) + if is_unanswerable: + continue + # ------------------ + + raw_question = qa_item.get("question", "") + question_id = qa_item.get("question_id", "") + # Append paper title to question for easier retrieval + question = f'Based on the paper "{paper_title}", {raw_question}' + + gold_answers = [] + evidence_list = [] + answer_types = [] + answer_evidence_pairs = [] + + # Iterate through all annotator answers + for answer_wrapper in qa_item.get("answers", []): + answer_obj = answer_wrapper.get("answer", {}) + + current_answer = None + answer_type = self._get_answer_type(answer_obj) + + # Process different answer types + if answer_obj.get("unanswerable", False): + current_answer = "Not mentioned" + gold_answers.append(current_answer) + else: + extractive_spans = answer_obj.get("extractive_spans", []) + free_form_answer = answer_obj.get("free_form_answer", "") + yes_no = answer_obj.get("yes_no") + + if extractive_spans: + valid_spans = [span.strip() for span in extractive_spans if span and span.strip()] + if valid_spans: + combined_answer = "; ".join(valid_spans) + gold_answers.append(combined_answer) + current_answer = combined_answer + else: + current_answer = None + elif free_form_answer and free_form_answer.strip(): + current_answer = free_form_answer.strip() + gold_answers.append(current_answer) + elif yes_no is not None: + current_answer = "Yes" if yes_no else "No" + gold_answers.append(current_answer) + + # Collect evidence text + current_evidence = [] + evidence = answer_obj.get("evidence", []) + for ev in evidence: + if ev and ev.strip(): + current_evidence.append(ev) + if ev not in evidence_list: + evidence_list.append(ev) + + # Record answer type (deduplicated) + if answer_type not in answer_types: + answer_types.append(answer_type) + + # Save answer-evidence correspondence + if current_answer: + answer_evidence_pairs.append({ + "answer": current_answer, + "evidence": current_evidence, + "answer_type": answer_type + }) + + # If no answers, default to "Not mentioned" + if not gold_answers: + gold_answers = ["Not mentioned"] + + # Deduplicate (preserve order) + gold_answers = list(dict.fromkeys(gold_answers)) + + qa_pairs.append(StandardQA( + question=question, + gold_answers=gold_answers, + evidence=evidence_list, + category=None, + metadata={ + "question_id": question_id, + "answer_types": answer_types, + "answer_evidence_pairs": answer_evidence_pairs + } + )) + + standard_samples.append(StandardSample( + sample_id=paper_id, + qa_pairs=qa_pairs + )) + + return standard_samples + + def _get_answer_type(self, answer_obj: Dict[str, Any]) -> str: + """ + Determine answer type from answer object. + + Args: + answer_obj: Answer object, containing extractive_spans, free_form_answer, yes_no, etc. + + Returns: + str: Answer type, possible values: + - "unanswerable": cannot answer + - "extractive": extractive answer + - "free_form": free-form answer + - "yes_no": yes/no answer + - "unknown": unknown type + """ + if answer_obj.get("unanswerable", False): + return "unanswerable" + if answer_obj.get("extractive_spans"): + return "extractive" + if answer_obj.get("free_form_answer", "").strip(): + return "free_form" + if answer_obj.get("yes_no") is not None: + return "yes_no" + return "unknown" + + def _convert_paper_to_markdown(self, paper_id: str, paper_data: Dict[str, Any]) -> str: + """ + Convert Qasper paper structure to Markdown string. + + Conversion format: + ```markdown + # {title} + Paper ID: {paper_id} + + ## Abstract + {abstract} + + ## Section Name 1 + {paragraph 1} + {paragraph 2} + + ## Section Name 2 + ... + + ## Figures and Tables + ### Figure 1 + Caption: {caption} + File: {filename} + + ### Table 1 + Caption: {caption} + File: {filename} + ``` + + Args: + paper_id: Paper ID + paper_data: Paper data, containing title, abstract, full_text, figures_and_tables + + Returns: + str: Markdown formatted paper content + """ + md_lines = [] + + # Title + title = paper_data.get("title", "Unknown Title") + md_lines.append(f"# {title}") + md_lines.append(f"Paper ID: {paper_id}\n") + + # Abstract + abstract = paper_data.get("abstract", "") + if abstract: + md_lines.append("## Abstract") + md_lines.append(abstract) + md_lines.append("") + + # Main text sections + full_text = paper_data.get("full_text", []) + for section in full_text: + section_name = section.get("section_name", "") + paragraphs = section.get("paragraphs", []) + + if section_name: + md_lines.append(f"## {section_name}") + + for para in paragraphs: + if para and para.strip(): + md_lines.append(para.strip()) + md_lines.append("") + + # Figure and table information + figures_and_tables = paper_data.get("figures_and_tables", []) + if figures_and_tables: + md_lines.append("## Figures and Tables") + for idx, fig in enumerate(figures_and_tables, 1): + caption = fig.get("caption", "") + file_name = fig.get("file", "") + + # Determine if figure or table based on filename or caption + if "Figure" in file_name or "figure" in caption.lower(): + md_lines.append(f"### Figure {idx}") + else: + md_lines.append(f"### Table {idx}") + + if caption: + md_lines.append(f"Caption: {caption}") + if file_name: + md_lines.append(f"File: {file_name}") + md_lines.append("") + + return "\n".join(md_lines) + + def build_prompt(self, qa: StandardQA, context_blocks: List[str]) -> tuple[str, Dict[str, Any]]: + context_text = "\n\n".join(context_blocks) if context_blocks else "No relevant context found." + + answer_types = qa.metadata.get("answer_types", []) + primary_type = answer_types[0] if answer_types else None + + category_instruction = CATEGORY_INSTRUCTIONS.get(primary_type, "") + + if category_instruction: + full_prompt = f"""{context_text} + +{category_instruction} + +{MISSING_RULE} + +Question: {qa.question} + +Answer:""" + else: + full_prompt = f"""{context_text} + +{MISSING_RULE} + +Question: {qa.question} + +Answer:""" + + meta = { + "question_id": qa.metadata.get("question_id", ""), + "answer_types": answer_types + } + return full_prompt, meta + + def post_process_answer(self, qa: StandardQA, raw_answer: str, meta: Dict[str, Any]) -> str: + """ + Post-process raw answer generated by LLM. + + Current implementation only strips leading/trailing whitespace. + + Args: + qa: Standardized QA object + raw_answer: Raw answer generated by LLM + meta: Metadata dictionary + + Returns: + str: Processed answer + """ + return raw_answer.strip() diff --git a/benchmark/RAG/src/adapters/syllabusqa_adapter.py b/benchmark/RAG/src/adapters/syllabusqa_adapter.py new file mode 100644 index 00000000..d0285037 --- /dev/null +++ b/benchmark/RAG/src/adapters/syllabusqa_adapter.py @@ -0,0 +1,464 @@ +# src/adapters/syllabusqa_adapter.py +""" +SyllabusQA Dataset Adapter + +SyllabusQA is a syllabus QA dataset containing 39 syllabi and 5078 questions. +Each question is about a specific syllabus, with answer types including: +- single factual: single factual question +- multi factual: multi factual question +- single reasoning: single reasoning question +- multi reasoning: multi reasoning question +- summarization: summarization question +- yes/no: yes/no question +- no answer: no answer can be found in the syllabus + +Dataset characteristics: +1. Each question has only one answer +2. Each answer has corresponding answer_span (evidence span) +3. reasoning type questions have reasoning steps +4. Original documents are in docx format + +Adapter functions: +- data_prepare: Convert docx to Markdown format +- load_and_transform: Parse QA data, category stores question type +- build_prompt: Build QA prompt +- post_process_answer: Post-process LLM output +""" + +import json +import os +import csv +from typing import List, Dict, Any + +from .base import BaseAdapter, StandardDoc, StandardSample, StandardQA + +# Rule for when answer cannot be found +MISSING_RULE = "If no information is available to answer the question, write 'Not mentioned'." + +# Specific instructions for different categories +CATEGORY_INSTRUCTIONS = { + "single factual": """Extract the single factual answer from the syllabus. +- Use EXACT wording from context when possible +- Provide concise, direct answer +- Do NOT add extra info or explanation""", + + "multi factual": """Extract multiple factual answers from the syllabus. +- Use EXACT wording from context when possible +- List items separated by commas +- Include all relevant facts""", + + "single reasoning": """Answer using simple logical reasoning based on the syllabus. +- Use ONLY facts from context +- Make clear, direct conclusion +- Do NOT explain reasoning +- Do NOT invent information""", + + "multi reasoning": """Answer using reasoning based on the syllabus. +- Use ONLY facts from context +- Do NOT invent information""", + + "summarization": """Summarize relevant information from the syllabus. +- Provide concise summary covering key points +- Use wording from syllabus when possible +- Include all important details""", + + "yes/no": """Answer Yes/No question based on the syllabus. +- First respond "Yes" or "No" +- Do NOT add explanation +- Use ONLY info from context +- Do NOT invent information""" +} + + +class SyllabusQAAdapter(BaseAdapter): + """ + Adapter specifically for processing SyllabusQA dataset. + + Converts syllabi (docx) to Markdown documents, + and converts QA data to standardized StandardSample format. + + Attributes: + raw_file_path: Raw CSV data file path + syllabus_dir: docx file directory path + logger: Logger + """ + + def __init__(self, raw_file_path: str, **kwargs): + """ + Initialize SyllabusQAAdapter. + + Args: + raw_file_path: Raw data file path (CSV or merged JSON) + **kwargs: Other parameters (ignored, for compatibility) + """ + super().__init__(raw_file_path) + # docx file directory, defaults to syllabi subdirectory under data directory + if os.path.isdir(raw_file_path): + self.syllabus_dir = os.path.join(raw_file_path, 'syllabi') + else: + self.syllabus_dir = os.path.join(os.path.dirname(raw_file_path), 'syllabi') + + def data_prepare(self, doc_dir: str) -> List[StandardDoc]: + """ + Load raw docx files and convert to OpenViking-friendly format. + + Only process syllabus documents mentioned in CSV, avoid processing irrelevant documents. + Requires python-docx library to parse docx files. + + Args: + doc_dir: Document output directory path + + Returns: + List[StandardDoc]: List of standardized document objects + + Raises: + FileNotFoundError: syllabus directory not found + """ + if not os.path.exists(self.syllabus_dir): + raise FileNotFoundError(f"Syllabus directory not found: {self.syllabus_dir}") + + res: List[StandardDoc] = [] + os.makedirs(doc_dir, exist_ok=True) + + # Get list of syllabus_name mentioned in CSV + required_syllabi = self._get_required_syllabi() + self.logger.info(f"[SyllabusQAAdapter] Required syllabi from CSV: {len(required_syllabi)}") + + # Get all docx files + docx_files = [f for f in os.listdir(self.syllabus_dir) if f.endswith('.docx')] + + for docx_file in docx_files: + syllabus_id = docx_file.replace('.docx', '') + + # Only process syllabi mentioned in CSV + if syllabus_id not in required_syllabi: + continue + + docx_path = os.path.join(self.syllabus_dir, docx_file) + + try: + # Convert docx to Markdown + doc_content = self._convert_docx_to_markdown(docx_path) + + doc_path = os.path.join(doc_dir, f"{syllabus_id}_doc.md") + with open(doc_path, "w", encoding="utf-8") as f: + f.write(doc_content) + res.append(StandardDoc(syllabus_id, doc_path)) + except Exception as e: + self.logger.error(f"[syllabusqa adapter] doc:{syllabus_id} prepare error {e}") + # If python-docx not installed, try using plain text + if "No module named 'docx'" in str(e): + self.logger.warning("python-docx not installed, skipping docx conversion") + break + raise e + + self.logger.info(f"[SyllabusQAAdapter] Processed {len(res)} syllabus documents") + return res + + def _get_required_syllabi(self) -> set: + """ + Get list of syllabus_name mentioned in CSV. + + Returns: + set: syllabus_name set + """ + required = set() + + # Determine data source type + if self.raw_file_path.endswith('.csv'): + csv_files = [self.raw_file_path] + elif os.path.isdir(self.raw_file_path): + csv_files = [os.path.join(self.raw_file_path, f) + for f in os.listdir(self.raw_file_path) + if f.endswith('.csv')] + else: + return required + + for csv_file in csv_files: + if not os.path.exists(csv_file): + continue + with open(csv_file, 'r', encoding='utf-8') as f: + import csv + reader = csv.DictReader(f) + for row in reader: + syllabus_name = row.get('syllabus_name', '') + if syllabus_name: + required.add(syllabus_name) + + return required + + def _convert_docx_to_markdown(self, docx_path: str) -> str: + """ + Convert docx file to Markdown string. + + Args: + docx_path: docx file path + + Returns: + str: Markdown formatted content + """ + try: + from docx import Document + except ImportError: + raise ImportError("python-docx is required. Install with: pip install python-docx") + + doc = Document(docx_path) + md_lines = [] + + # Extract filename as title + filename = os.path.basename(docx_path).replace('.docx', '') + md_lines.append(f"# {filename}") + md_lines.append("") + + # Iterate through all paragraphs + for para in doc.paragraphs: + text = para.text.strip() + if text: + # Check if heading style + if para.style.name.startswith('Heading'): + level = para.style.name.replace('Heading ', '') + try: + level_num = int(level) + md_lines.append(f"{'#' * level_num} {text}") + except ValueError: + md_lines.append(f"## {text}") + else: + md_lines.append(text) + md_lines.append("") + + # Extract tables + for table in doc.tables: + md_lines.append("## Table") + for row in table.rows: + cells = [cell.text.strip() for cell in row.cells] + md_lines.append("| " + " | ".join(cells) + " |") + md_lines.append("") + + return "\n".join(md_lines) + + def load_and_transform(self) -> List[StandardSample]: + """ + Load raw CSV data and convert to standardized StandardSample object list. + + Processing logic: + 1. Read CSV file (supports single CSV or directory) + 2. Group by syllabus_name + 3. Format question as "Based on the syllabus \"{syllabus_name}\", {question}" + 4. category stores question_type + 5. answer_span as evidence + + Returns: + List[StandardSample]: List of standardized sample objects + + Raises: + FileNotFoundError: Raw data file not found + """ + # Determine if CSV file or JSON file + if self.raw_file_path.endswith('.json'): + return self._load_from_json() + elif self.raw_file_path.endswith('.csv'): + return self._load_from_csv([self.raw_file_path]) + elif os.path.isdir(self.raw_file_path): + # Directory, find all CSV files + csv_files = [os.path.join(self.raw_file_path, f) + for f in os.listdir(self.raw_file_path) + if f.endswith('.csv')] + return self._load_from_csv(csv_files) + else: + raise FileNotFoundError(f"Unsupported file format: {self.raw_file_path}") + + def _load_from_json(self) -> List[StandardSample]: + """ + Load data from JSON file. + """ + if not os.path.exists(self.raw_file_path): + raise FileNotFoundError(f"Raw data file not found: {self.raw_file_path}") + + with open(self.raw_file_path, 'r', encoding='utf-8') as f: + data = json.load(f) + + standard_samples = [] + + for syllabus_name, qa_list in data.items(): + qa_pairs = [] + + for qa_item in qa_list: + question = qa_item.get("question", "") + answer = qa_item.get("answer", "") + question_type = qa_item.get("question_type", "") + qa_id = qa_item.get("id", "") + + # Skip "no answer" type questions as RAG results cannot be evaluated + if question_type == "no answer": + continue + + # Collect answer_span as evidence + evidence = [] + for i in range(1, 6): + span = qa_item.get(f"answer_span_{i}", "") + if span and span.strip(): + evidence.append(span.strip()) + + # Collect reasoning_steps, also as evidence (for reasoning type questions) + reasoning_steps = [] + for i in range(1, 6): + step = qa_item.get(f"reasoning_step_{i}", "") + if step and step.strip(): + reasoning_steps.append(step.strip()) + # reasoning_steps also added to evidence for recall calculation + if step.strip() not in evidence: + evidence.append(step.strip()) + + # Format question + formatted_question = f'Based on the syllabus "{syllabus_name}", {question}' + + qa_pairs.append(StandardQA( + question=formatted_question, + gold_answers=[answer] if answer else ["Not mentioned"], + evidence=evidence, + category=question_type, + metadata={ + "id": qa_id, + "reasoning_steps": reasoning_steps + } + )) + + # Only add samples with QA pairs to result + if qa_pairs: + standard_samples.append(StandardSample( + sample_id=syllabus_name, + qa_pairs=qa_pairs + )) + + return standard_samples + + def _load_from_csv(self, csv_files: List[str]) -> List[StandardSample]: + """ + Load data from CSV files. + + Args: + csv_files: List of CSV file paths + + Returns: + List[StandardSample]: List of standardized sample objects + """ + # Group by syllabus_name + syllabus_qa_map: Dict[str, List] = {} + + for csv_file in csv_files: + if not os.path.exists(csv_file): + self.logger.warning(f"CSV file not found: {csv_file}") + continue + + with open(csv_file, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + for row in reader: + syllabus_name = row.get('syllabus_name', '') + if syllabus_name not in syllabus_qa_map: + syllabus_qa_map[syllabus_name] = [] + syllabus_qa_map[syllabus_name].append(row) + + standard_samples = [] + + for syllabus_name, qa_list in syllabus_qa_map.items(): + qa_pairs = [] + + for qa_item in qa_list: + question = qa_item.get("question", "") + answer = qa_item.get("answer", "") + question_type = qa_item.get("question_type", "") + qa_id = qa_item.get("id", "") + + # Skip "no answer" type questions as RAG results cannot be evaluated + if question_type == "no answer": + continue + + # Collect answer_span as evidence + evidence = [] + for i in range(1, 6): + span = qa_item.get(f"answer_span_{i}", "") + if span and span.strip(): + evidence.append(span.strip()) + + # Collect reasoning_steps, also as evidence (for reasoning type questions) + reasoning_steps = [] + for i in range(1, 6): + step = qa_item.get(f"reasoning_step_{i}", "") + if step and step.strip(): + reasoning_steps.append(step.strip()) + # reasoning_steps also added to evidence for recall calculation + if step.strip() not in evidence: + evidence.append(step.strip()) + + # Format question + formatted_question = f'Based on the syllabus "{syllabus_name}", {question}' + + qa_pairs.append(StandardQA( + question=formatted_question, + gold_answers=[answer] if answer else ["Not mentioned"], + evidence=evidence, + category=question_type, + metadata={ + "id": qa_id, + "reasoning_steps": reasoning_steps + } + )) + + # Only add samples with QA pairs to result + if qa_pairs: + standard_samples.append(StandardSample( + sample_id=syllabus_name, + qa_pairs=qa_pairs + )) + + return standard_samples + + def build_prompt(self, qa: StandardQA, context_blocks: List[str]) -> tuple[str, Dict[str, Any]]: + """ + Build complete prompt to send to LLM. + + Prompt structure: + 1. Context content (retrieved document fragments) + 2. Category-specific instructions + 3. Rule for when answer cannot be found + 4. Question + + Args: + qa: Standardized QA object + context_blocks: List of retrieved context text blocks + + Returns: + tuple[str, Dict[str, Any]]: + - Complete prompt string + - Metadata dictionary, containing id + """ + eff_q = qa.question + category = qa.category + + category_instruction = CATEGORY_INSTRUCTIONS.get(category, "") + + context_text = "\n\n".join(context_blocks) + + if category_instruction: + full_prompt = f"{context_text}\n\n{category_instruction}\n\n{MISSING_RULE}\n\nQuestion: {eff_q}\n\nAnswer:" + else: + full_prompt = f"{context_text}\n\n{MISSING_RULE}\n\nQuestion: {eff_q}\n\nAnswer:" + + meta = {"id": qa.metadata.get("id", "")} + return full_prompt, meta + + def post_process_answer(self, qa: StandardQA, raw_answer: str, meta: Dict[str, Any]) -> str: + """ + Post-process raw answer generated by LLM. + + Current implementation only strips leading/trailing whitespace. + + Args: + qa: Standardized QA object + raw_answer: Raw answer generated by LLM + meta: Metadata dictionary + + Returns: + str: Processed answer + """ + return raw_answer.strip() diff --git a/benchmark/RAG/src/core/__init__.py b/benchmark/RAG/src/core/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/benchmark/RAG/src/core/judge_util.py b/benchmark/RAG/src/core/judge_util.py new file mode 100644 index 00000000..effaa3c4 --- /dev/null +++ b/benchmark/RAG/src/core/judge_util.py @@ -0,0 +1,164 @@ +import json +import re + +from langchain_core.messages import HumanMessage, SystemMessage + + +def llm_grader( + llm_client, + model: str, + question: str, + gold_answer: str or list, + response: str, + dataset_name: str = "Locomo" +) -> dict: + """ + Use an LLM as a judge to score a generated answer against a gold answer. + + Return format: + { + "score": int, # LoCoMo: 0 or 4; Qasper/Generic: 0~4 + "reasoning": str, # grading explanation or fallback parse info + "prompt_type": str # which prompt template was used + } + """ + + dataset_name_lower = (dataset_name or "").lower() + content = "" + score = 0 + reasoning = "No reasoning provided." + prompt_type = "Generic_0-4" + + # Handle case when gold_answer is a list + if isinstance(gold_answer, list): + gold_answer_str = " | ".join(gold_answer) + else: + gold_answer_str = gold_answer + + # ------------------------- + # 1) Route by dataset + # ------------------------- + if "locomo" in dataset_name_lower: + prompt_type = "Locomo_0or4" + + system_prompt = """ +You are an expert grader that determines if answers to questions match a gold standard answer +""" + + ACCURACY_PROMPT = f""" +Your task is to label an answer to a question by assigning a score of 4 or 0. You will be given the following data: +(1) a question (posed by one user to another user), +(2) a 'gold' (ground truth) answer, +(3) a generated answer + +which you will score as 4 or 0. +The point of the question is to ask about something one user should know about the other user based on their prior conversations. +The gold answer will usually be a concise and short answer that includes the referenced topic, for example: +Question: Do you remember what I got the last time I went to Hawaii? +Gold answer: A shell necklace +The generated answer might be much longer, but you should be generous with your grading - as long as it touches on the same topic as the gold answer, it should be counted as correct. +For time related questions, the gold answer will be a specific date, month, year, etc. The generated answer might be much longer or use relative time references (like "last Tuesday" or "next month"), but you should be generous with your grading - as long as it refers to the same date or time period as the gold answer, it should be counted as correct. Even if the format differs (e.g., "May 7th" vs "7 May"), consider it correct if it's the same date. + +Scoring rule: +- Output score 4 if the generated answer should be considered CORRECT. +- Output score 0 if the generated answer should be considered WRONG. + +Now it's time for the real question: +Question: {question} +Gold answer: {gold_answer} +Generated answer: {response} + +First, provide a short (one sentence) explanation of your reasoning. +Respond with JSON only: {{"score": 4 or 0, "reasoning": "your explanation"}} +""" + + else: + prompt_type = "Generic_0-4" + + system_prompt = """ +You are an expert evaluator scoring how well an AI-generated answer matches a gold standard (ground truth). +""" + + ACCURACY_PROMPT = f""" +Please score the Generated Answer against the Gold Answers on a scale of 0 to 4. + +[Evaluation Rubric] +- Score 4 (Perfect): Fully and accurately captures the core meaning and key facts of any of the Gold Answers. Additional relevant explanation or context is acceptable and does NOT reduce the score, as long as it is consistent with and does not contradict the Gold Answers. Minor differences in wording, capitalization, punctuation, or phrasing are acceptable if the core meaning is preserved. +- Score 3 (Good): Correctly captures the main answer and most key facts, but has minor issues such as slight imprecision, small omissions of non-critical details, or wording that is somewhat vague or ambiguous. The overall answer is still clearly correct. +- Score 2 (Partial): Partially correct, but missing at least one important fact, condition, or detail needed for a fully correct answer. The answer is related to the correct topic, but is incomplete or insufficient. +- Score 1 (Poor): Mostly incorrect, seriously incomplete, or only weakly related to the Gold Answers. +- Score 0 (Wrong): Incorrect, contradictory to the Gold Answers, or contains fabricated / hallucinated core content. + +Important Notes: +- Gold answers are multiple possible correct answers separated by " | ". The generated answer only needs to match any one of them. +- The gold answers may be concise, but the generated answer can be longer and include additional explanations - this is acceptable for Score 4 as long as the core information is correct. +- Do NOT penalize for additional relevant information that doesn't contradict the gold answers. Examples of acceptable extra information: titles ("King Padella" vs "Padella"), locations ("Paflagonia" vs "the capital of Paflagonia"), or additional context that supports the answer. +- Only penalize for actual incorrect information, missing key facts, or contradictions. +- Ignore minor differences in capitalization (e.g., "CRIM TARTARY" vs "Crim Tartary") or punctuation (e.g., with or without a period at the end). + +Question: {question} +Gold Answers: {gold_answer_str} +Generated Answer: {response} + +First, briefly explain the rating in 1 sentence. Then output the integer score. +Respond ONLY with a JSON object: {{"score": 0 to 4, "reasoning": "string"}} +""" + + messages = [ + SystemMessage(content=system_prompt), + HumanMessage(content=ACCURACY_PROMPT), + ] + + # ------------------------- + # 2) Unified invoke + parse + # ------------------------- + try: + resp = llm_client.invoke(messages) + content = resp.content if resp and hasattr(resp, "content") else "" + + result = json.loads(content) + score = int(result.get("score", 0)) + reasoning = result.get("reasoning", "No reasoning provided.") + + # Clamp score by dataset + if "locomo" in dataset_name_lower: + # LoCoMo only allows 0 or 4 + score = 4 if score == 4 else 0 + else: + # Other datasets allow 0~4 + score = max(0, min(4, score)) + + except Exception: + # ------------------------- + # 3) Unified fallback parse + # ------------------------- + text = (content or "").strip() + reasoning = ( + f"Parse fallback from raw output: {text}" + if text + else "Parse failed or model invocation failed. Defaulted to 0." + ) + + # First try: JSON-like score field + match = re.search(r'"score"\s*:\s*([0-4])', text) + if match: + score = int(match.group(1)) + else: + # Second try: any standalone integer 0~4 in text + match = re.search(r'\b([0-4])\b', text) + if match: + score = int(match.group(1)) + else: + score = 0 + + # Dataset-specific clamp + if "locomo" in dataset_name_lower: + score = 4 if score == 4 else 0 + else: + score = max(0, min(4, score)) + + return { + "score": score, + "reasoning": reasoning, + "prompt_type": prompt_type, + } diff --git a/benchmark/RAG/src/core/llm_client.py b/benchmark/RAG/src/core/llm_client.py new file mode 100644 index 00000000..c1c90f4a --- /dev/null +++ b/benchmark/RAG/src/core/llm_client.py @@ -0,0 +1,27 @@ +import time +from langchain_openai import ChatOpenAI +from langchain_core.messages import HumanMessage + + +class LLMClientWrapper: + def __init__(self, config: dict, api_key: str): + self.llm = ChatOpenAI( + model=config['model'], + temperature=config['temperature'], + api_key=api_key, + base_url=config['base_url'] + ) + self.retry_count = 3 + + def generate(self, prompt: str) -> str: + """Call LLM to generate answer with simple exponential backoff retry""" + last_err = None + for attempt in range(self.retry_count): + try: + resp = self.llm.invoke([HumanMessage(content=prompt)]) + return resp.content + except Exception as e: + last_err = e + time.sleep(1.5 * (attempt + 1)) + + return f"ERROR: {str(last_err)}" diff --git a/benchmark/RAG/src/core/logger.py b/benchmark/RAG/src/core/logger.py new file mode 100644 index 00000000..ab384730 --- /dev/null +++ b/benchmark/RAG/src/core/logger.py @@ -0,0 +1,26 @@ +import logging +import os + + +def setup_logging(log_file): + os.makedirs(os.path.dirname(log_file), exist_ok=True) + + logger = logging.getLogger("Benchmark") + logger.setLevel(logging.INFO) + logger.handlers = [] + + formatter = logging.Formatter("%(asctime)s | %(levelname)-7s | %(message)s") + + fh = logging.FileHandler(log_file, mode="a", encoding="utf-8") + fh.setFormatter(formatter) + logger.addHandler(fh) + + sh = logging.StreamHandler() + sh.setFormatter(formatter) + logger.addHandler(sh) + + return logger + + +def get_logger() -> logging.Logger: + return logging.getLogger("Benchmark") diff --git a/benchmark/RAG/src/core/metrics.py b/benchmark/RAG/src/core/metrics.py new file mode 100644 index 00000000..d40d8de1 --- /dev/null +++ b/benchmark/RAG/src/core/metrics.py @@ -0,0 +1,86 @@ +import re +import string +import collections +from typing import List + + +class MetricsCalculator: + @staticmethod + def normalize_answer(s): + """Normalize answer text: remove punctuation, convert to lowercase, remove articles""" + s = str(s).replace(',', "") + def remove_articles(text): return re.sub(r'\b(a|an|the|and)\b', ' ', text) + def white_space_fix(text): return ' '.join(text.split()) + def remove_punc(text): + exclude = set(string.punctuation) + return ''.join(ch for ch in text if ch not in exclude) + return white_space_fix(remove_articles(remove_punc(s.lower()))) + + @staticmethod + def calculate_f1(prediction: str, ground_truth: str) -> float: + pred_tokens = MetricsCalculator.normalize_answer(prediction).split() + truth_tokens = MetricsCalculator.normalize_answer(ground_truth).split() + common = collections.Counter(pred_tokens) & collections.Counter(truth_tokens) + num_same = sum(common.values()) + if num_same == 0: return 0.0 + precision = 1.0 * num_same / len(pred_tokens) + recall = 1.0 * num_same / len(truth_tokens) + return (2 * precision * recall) / (precision + recall) + + @staticmethod + def check_refusal(text: str) -> bool: + refusals = ["not mentioned", "no information", "cannot be answered", "none", "unknown", "don't know"] + return any(r in text.lower() for r in refusals) + + @staticmethod + def check_recall(retrieved_texts: List[str], evidence_list: List[str], soft_threshold: float = 0.8, min_soft_match_tokens: int = 4) -> float: + """ + Calculate retrieval recall combining strict substring matching with dynamic token-based soft matching. + + Approach: + - Combine and preprocess: concatenate multiple retrieved text chunks into a single string. + - Strict matching first: check if evidence exists as a complete substring in the combined retrieved text. + - Length blocking mechanism: calculate effective token count of evidence. If below threshold (e.g., short IDs or entities), directly determine no hit after strict match failure, prohibiting soft matching. + - Soft matching fallback: for long text evidence, calculate token coverage in retrieved text, consider hit if threshold is met. + - Equal weighting: each evidence has equal weight, final score is hit count / total count. + + Args: + retrieved_texts: List[str], list of text chunks returned by retrieval module (required) + evidence_list: List[str], ground truth evidence list containing IDs or long text evidence (required) + soft_threshold: float, coverage threshold for soft matching to be considered a hit (optional, 0.0~1.0, default 0.8) + min_soft_match_tokens: int, minimum effective token count threshold allowing fallback to soft matching (optional, default 4. Short texts below this length require strict matching) + + Returns: + float, retrieval recall score, range 0.0 to 1.0 + """ + if not evidence_list: + return 0.0 + + combined_retrieved = " ".join(retrieved_texts) + + normalized_retrieved = MetricsCalculator.normalize_answer(combined_retrieved) + ret_tokens = set(normalized_retrieved.split()) + + hit_count = 0 + + for evidence in evidence_list: + if evidence in combined_retrieved: + hit_count += 1 + continue + + normalized_ev = MetricsCalculator.normalize_answer(evidence) + ev_tokens = set(normalized_ev.split()) + + if not ev_tokens: + continue + + if len(ev_tokens) < min_soft_match_tokens: + continue + + overlap_count = len(ev_tokens & ret_tokens) + coverage = overlap_count / len(ev_tokens) + + if coverage >= soft_threshold: + hit_count += 1 + + return hit_count / len(evidence_list) diff --git a/benchmark/RAG/src/core/monitor.py b/benchmark/RAG/src/core/monitor.py new file mode 100644 index 00000000..4c25fb30 --- /dev/null +++ b/benchmark/RAG/src/core/monitor.py @@ -0,0 +1,51 @@ +import threading +import time +from dataclasses import dataclass + + +@dataclass +class MonitorStats: + active_threads: int = 0 + completed_tasks: int = 0 + failed_tasks: int = 0 + total_tokens: int = 0 + start_time: float = 0.0 + + +class BenchmarkMonitor: + def __init__(self): + self._lock = threading.Lock() + self.stats = MonitorStats() + self.stats.start_time = time.time() + + def worker_start(self): + with self._lock: + self.stats.active_threads += 1 + + def worker_end(self, tokens=0, success=True): + with self._lock: + self.stats.active_threads -= 1 + self.stats.completed_tasks += 1 + self.stats.total_tokens += tokens + if not success: + self.stats.failed_tasks += 1 + + def get_status_dict(self): + """Return real-time status dictionary for tqdm progress bar display""" + elapsed = time.time() - self.stats.start_time + qps = self.stats.completed_tasks / elapsed if elapsed > 0 else 0 + + tokens = self.stats.total_tokens + if tokens > 1_000_000: + token_str = f"{tokens/1_000_000:.1f}M" + elif tokens > 1_000: + token_str = f"{tokens/1_000:.1f}k" + else: + token_str = str(tokens) + + return { + "Active": self.stats.active_threads, + "QPS": f"{qps:.2f}", + "Tokens": token_str, + "Errs": self.stats.failed_tasks + } diff --git a/benchmark/RAG/src/core/vector_store.py b/benchmark/RAG/src/core/vector_store.py new file mode 100644 index 00000000..39af7a34 --- /dev/null +++ b/benchmark/RAG/src/core/vector_store.py @@ -0,0 +1,116 @@ +import os +import time +from typing import List +import sys +from pathlib import Path + +sys.path.append(str(Path(__file__).parent.parent)) + +from adapters.base import StandardDoc, StandardSample +import tiktoken +import openviking as ov + + +class VikingStoreWrapper: + def __init__(self, store_path: str): + self.store_path = store_path + if not os.path.exists(store_path): + os.makedirs(store_path) + + self.client = ov.SyncOpenViking(path=store_path) + + try: + self.enc = tiktoken.get_encoding("cl100k_base") + except Exception as e: + print(f"[Warning] tiktoken init failed: {e}") + self.enc = None + + def count_tokens(self, text: str) -> int: + if not text or not self.enc: + return 0 + return len(self.enc.encode(str(text))) + + def ingest(self, samples: List[StandardDoc], max_workers=10, monitor=None, ingest_mode="per_file") -> dict: + start_time = time.time() + total_input_tokens = 0 + total_output_tokens = 0 + total_embedding_tokens = 0 + + if not samples: + return { + "time": time.time() - start_time, + "input_tokens": 0, + "output_tokens": 0 + } + + if ingest_mode == "directory": + def find_common_ancestor(paths): + if not paths: + return None + path_components = [p.split(os.sep) for p in paths] + min_length = min(len(components) for components in path_components) + common = [] + for i in range(min_length): + current_component = path_components[0][i] + if all(components[i] == current_component for components in path_components): + common.append(current_component) + else: + break + if common: + return os.sep.join(common) + return None + + doc_paths = [os.path.abspath(s.doc_path) for s in samples] + common_ancestor = find_common_ancestor(doc_paths) + + if common_ancestor: + result = self.client.add_resource(common_ancestor, wait=True, telemetry=True) + telemetry = result.get("telemetry", {}) + summary = telemetry.get("summary", {}) + tokens = summary.get("tokens", {}) + llm_tokens = tokens.get("llm", {}) + embedding_tokens = tokens.get("embedding", {}) + total_input_tokens = llm_tokens.get("input", 0) + total_output_tokens = llm_tokens.get("output", 0) + total_embedding_tokens = embedding_tokens.get("total", 0) + else: + for sample in samples: + result = self.client.add_resource(sample.doc_path, wait=True, telemetry=True) + telemetry = result.get("telemetry", {}) + summary = telemetry.get("summary", {}) + tokens = summary.get("tokens", {}) + llm_tokens = tokens.get("llm", {}) + embedding_tokens = tokens.get("embedding", {}) + total_input_tokens += llm_tokens.get("input", 0) + total_output_tokens += llm_tokens.get("output", 0) + total_embedding_tokens += embedding_tokens.get("total", 0) + else: + for sample in samples: + result = self.client.add_resource(sample.doc_path, wait=True, telemetry=True) + telemetry = result.get("telemetry", {}) + summary = telemetry.get("summary", {}) + tokens = summary.get("tokens", {}) + llm_tokens = tokens.get("llm", {}) + embedding_tokens = tokens.get("embedding", {}) + total_input_tokens += llm_tokens.get("input", 0) + total_output_tokens += llm_tokens.get("output", 0) + total_embedding_tokens += embedding_tokens.get("total", 0) + + return { + "time": time.time() - start_time, + "input_tokens": total_input_tokens, + "output_tokens": total_output_tokens, + "embedding_tokens": total_embedding_tokens + } + + def retrieve(self, query: str, topk: int, target_uri: str = "viking://resources"): + """Execute retrieval""" + return self.client.find(query=query, limit=topk, target_uri=target_uri) + + def read_resource(self, uri: str) -> str: + """Read resource content""" + return str(self.client.read(uri)) + + def clear(self): + """Clear the store""" + self.client.rm("viking://resources", recursive=True) diff --git a/benchmark/RAG/src/pipeline.py b/benchmark/RAG/src/pipeline.py new file mode 100644 index 00000000..22a8410d --- /dev/null +++ b/benchmark/RAG/src/pipeline.py @@ -0,0 +1,345 @@ +import os +import json +import time +import random +import re +from concurrent.futures import ThreadPoolExecutor, as_completed +from tqdm import tqdm +from pathlib import Path +import sys + +sys.path.append(str(Path(__file__).parent)) + +from adapters.base import BaseAdapter +from core.logger import get_logger +from core.vector_store import VikingStoreWrapper +from core.monitor import BenchmarkMonitor +from core.metrics import MetricsCalculator +from core.judge_util import llm_grader + + +class BenchmarkPipeline: + def __init__(self, config, adapter: BaseAdapter, vector_db: VikingStoreWrapper, llm): + self.config = config + self.adapter = adapter + self.db = vector_db + self.llm = llm + self.logger = get_logger() + self.monitor = BenchmarkMonitor() + + self.output_dir = self.config['paths']['output_dir'] + if not os.path.exists(self.output_dir): + os.makedirs(self.output_dir, exist_ok=True) + self.generated_file = os.path.join(self.output_dir, "generated_answers.json") + self.eval_file = os.path.join(self.output_dir, "qa_eval_detailed_results.json") + self.report_file = os.path.join(self.output_dir, "benchmark_metrics_report.json") + + self.metrics_summary = { + "insertion": {"time": 0, "input_tokens": 0, "output_tokens": 0, "embedding_tokens": 0}, + "deletion": {"time": 0, "input_tokens": 0, "output_tokens": 0, "embedding_tokens": 0} + } + + def run_generation(self): + """Step 1: Data Preparation""" + self.logger.info(">>> Stage: Ingestion & Generation") + doc_dir = self.config['paths'].get('doc_output_dir') + if not doc_dir: + doc_dir = os.path.join(self.output_dir, "docs") + + try: + doc_info = self.adapter.data_prepare(doc_dir) + except Exception as e: + self.logger.exception(f"Data preparation failed: {e}") + exit(1) + + skip_ingestion = self.config['execution'].get('skip_ingestion', False) + + if skip_ingestion: + self.logger.info(f"Skipping Ingestion. Using existing docs at: {doc_dir}") + if not os.path.exists(doc_dir): + self.logger.warning(f"Warning: Doc directory {doc_dir} not found, but ingestion is skipped.") + self.metrics_summary["insertion"] = {"time": 0, "input_tokens": 0, "output_tokens": 0, "embedding_tokens": 0} + else: + ingest_workers = self.config['execution'].get('ingest_workers', 10) + ingest_mode = self.config['execution'].get('ingest_mode', 'per_file') + + mode_desc = { + 'directory': 'Unified directory mode', + 'per_file': 'Per-file mode' + } + self.logger.info(f"Ingestion mode: {ingest_mode} ({mode_desc.get(ingest_mode, 'Unknown mode')})") + self.logger.info(f"Number of documents: {len(doc_info)}") + + ingest_stats = self.db.ingest( + doc_info, + max_workers=ingest_workers, + monitor=self.monitor, + ingest_mode=ingest_mode + ) + self.metrics_summary["insertion"] = ingest_stats + self.logger.info(f"Insertion finished. Time: {ingest_stats['time']:.2f}s") + + self._update_report({ + "Insertion Efficiency (Total Dataset)": { + "Total Insertion Time (s)": self.metrics_summary["insertion"]["time"], + "Total Input Tokens": self.metrics_summary["insertion"]["input_tokens"], + "Total Output Tokens": self.metrics_summary["insertion"]["output_tokens"], + "Total Embedding Tokens": self.metrics_summary["insertion"].get("embedding_tokens", 0) + } + }) + + samples = self.adapter.load_and_transform() + tasks = self._prepare_tasks(samples) + results_map = {} + max_workers = self.config['execution']['max_workers'] + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + future_to_task = { + executor.submit(self._process_generation_task, task): task + for task in tasks + } + + pbar = tqdm(total=len(tasks), desc="Generating Answers", unit="task") + for future in as_completed(future_to_task): + task = future_to_task[future] + try: + res = future.result() + results_map[res['_global_index']] = res + except Exception as e: + self.logger.error(f"Generation failed for task {task['id']}: {e}") + self.monitor.worker_end(success=False) + pbar.set_postfix(self.monitor.get_status_dict()) + pbar.update(1) + pbar.close() + + sorted_results = [results_map[i] for i in sorted(results_map.keys())] + dataset_name = self.config.get('dataset_name', 'Unknown_Dataset') + save_data = { + "summary": {"dataset": dataset_name, "total_queries": len(sorted_results)}, + "results": sorted_results + } + total = len(sorted_results) + if total > 0: + self._update_report({ + "Query Efficiency (Average Per Query)": { + "Average Retrieval Time (s)": sum(r['retrieval']['latency_sec'] for r in sorted_results) / total, + "Average Input Tokens": sum(r['token_usage']['total_input_tokens'] for r in sorted_results) / total, + "Average Output Tokens": sum(r['token_usage']['llm_output_tokens'] for r in sorted_results) / total, + } + } + ) + with open(self.generated_file, "w", encoding="utf-8") as f: + json.dump(save_data, f, indent=2, ensure_ascii=False) + + def run_evaluation(self): + """Step 4: Evaluation""" + self.logger.info(">>> Stage: Evaluation") + + if not os.path.exists(self.generated_file): + self.logger.error("Generated answers file not found.") + return + + with open(self.generated_file, "r", encoding="utf-8") as f: + data = json.load(f) + items = data.get("results", []) + + eval_items = items + eval_results_map = {} + + with ThreadPoolExecutor(max_workers=self.config['execution']['max_workers']) as executor: + future_to_item = { + executor.submit(self._process_evaluation_task, item): item + for item in eval_items + } + + pbar = tqdm(total=len(eval_items), desc="Evaluating", unit="item") + for future in as_completed(future_to_item): + try: + res = future.result() + eval_results_map[res['_global_index']] = res + except Exception as e: + self.logger.error(f"Evaluation failed: {e}") + pbar.update(1) + pbar.close() + + eval_records = list(eval_results_map.values()) + total = len(eval_records) + + with open(self.eval_file, "w", encoding="utf-8") as f: + json.dump({"results": eval_records}, f, indent=2, ensure_ascii=False) + + if total > 0: + self._update_report({ + "Dataset": self.config.get('dataset_name', 'Unknown_Dataset'), + "Total Queries Evaluated": total, + "Performance Metrics": { + "Average F1 Score": sum(r['metrics']['F1'] for r in eval_records) / total, + "Average Recall": sum(r['metrics']['Recall'] for r in eval_records) / total, + "Average Accuracy (Hit 0-4)": sum(r['metrics']['Accuracy'] for r in eval_records) / total, + "Average Accuracy (normalization)": (sum(r['metrics']['Accuracy'] for r in eval_records) / total)/4, + } + }) + + def run_deletion(self): + """Step 5: Cleanup""" + self.logger.info(">>> Stage: Deletion") + start_time = time.time() + self.db.clear() + duration = time.time() - start_time + self.metrics_summary["deletion"] = {"time": duration, "input_tokens": 0, "output_tokens": 0} + self.logger.info(f"Deletion finished. Time: {duration:.2f}s") + + self._update_report({ + "Deletion Efficiency (Total Dataset)": { + "Total Deletion Time (s)": duration, + "Total Input Tokens": 0, + "Total Output Tokens": 0 + } + }) + + def _prepare_tasks(self, samples): + tasks = [] + global_idx = 0 + max_queries = self.config['execution'].get('max_queries') + for sample in samples: + for qa in sample.qa_pairs: + if max_queries is not None and global_idx >= max_queries: + break + tasks.append({"id": global_idx, "sample_id": sample.sample_id, "qa": qa}) + global_idx += 1 + if max_queries is not None and global_idx >= max_queries: + break + return tasks + + def _process_generation_task(self, task): + self.monitor.worker_start() + try: + qa = task['qa'] + + t0 = time.time() + # Get retrieval instruction from config, default to empty + retrieval_instruction = self.config['execution'].get('retrieval_instruction', '') + # Build enhanced query with instruction if provided + if retrieval_instruction: + enhanced_query = f"{retrieval_instruction} {qa.question}" + else: + enhanced_query = qa.question + search_res = self.db.retrieve(query=enhanced_query, topk=self.config['execution']['retrieval_topk']) + latency = time.time() - t0 + + retrieved_texts = [] + retrieved_uris = [] + context_blocks = [] + + for r in search_res.resources: + retrieved_uris.append(r.uri) + content = self.db.read_resource(r.uri) if getattr(r, 'level', 2) == 2 else f"{getattr(r, 'abstract', '')}\n{getattr(r, 'overview', '')}" + retrieved_texts.append(content) + clean = content[:8000] + context_blocks.append(clean) + + recall = MetricsCalculator.check_recall(retrieved_texts, qa.evidence) + + full_prompt, meta = self.adapter.build_prompt(qa, context_blocks) + + ans_raw = self.llm.generate(full_prompt) + + ans = self.adapter.post_process_answer(qa, ans_raw, meta) + + in_tokens = self.db.count_tokens(full_prompt) + self.db.count_tokens(qa.question) + out_tokens = self.db.count_tokens(ans) + self.monitor.worker_end(tokens=in_tokens + out_tokens) + + self.logger.info(f"[Query-{task['id']}] Q: {qa.question[:30]}... | Recall: {recall:.2f} | Latency: {latency:.2f}s") + + return { + "_global_index": task['id'], "sample_id": task['sample_id'], "question": qa.question, + "gold_answers": qa.gold_answers, "category": str(qa.category), "evidence": qa.evidence, + "retrieval": {"latency_sec": latency, "uris": retrieved_uris}, + "llm": {"final_answer": ans}, + "metrics": {"Recall": recall}, "token_usage": {"total_input_tokens": in_tokens, "llm_output_tokens": out_tokens} + } + except Exception as e: + self.monitor.worker_end(success=False) + raise e + + def _process_evaluation_task(self, item): + """ + Process a single evaluation task, computing F1 and Accuracy metrics. + + For multi-annotator scenarios (like Qasper dataset), a question may have multiple gold answers. + Evaluation logic: + - F1: Compute for each gold answer separately and take the maximum + - Accuracy: Pass all gold answers to LLM at once for comprehensive judgment + + This correctly handles multi-annotator scenarios while maintaining compatibility with single-answer datasets (like Locomo). + """ + ans, golds = item['llm']['final_answer'], item['gold_answers'] + + f1 = max((MetricsCalculator.calculate_f1(ans, gt) for gt in golds), default=0.0) + + dataset_name = self.config.get('dataset_name', 'Unknown_Dataset') + + eval_record = { + "score": 0.0, + "reasoning": "", + "prompt_type": "" + } + + try: + eval_res = llm_grader( + self.llm.llm, + self.config['llm']['model'], + item['question'], + golds, + ans, + dataset_name=dataset_name + ) + eval_record = eval_res + + except Exception as e: + self.logger.error(f"Grader error: {e}") + + if MetricsCalculator.check_refusal(ans) and any(MetricsCalculator.check_refusal(gt) for gt in golds): + f1 = 1.0 + eval_record["score"] = 4.0 + eval_record["reasoning"] = "System successfully identified Unanswerable/Refusal condition." + eval_record["prompt_type"] = "Heuristic_Refusal_Check" + + acc = eval_record["score"] + + item["metrics"].update({"F1": f1, "Accuracy": acc}) + + item["llm_evaluation"] = { + "prompt_used": eval_record["prompt_type"], + "reasoning": eval_record["reasoning"], + "normalized_score": acc + } + + detailed_info = ( + f"\n" + "="*60 + + f"\n[Query ID]: {item['_global_index']}" + f"\n[Question]: {item['question']}" + f"\n[Retrieved URIs]: {item['retrieval'].get('uris', [])}" + f"\n[LLM Answer]: {ans}" + f"\n[Gold Answer]: {golds}" + f"\n[Metrics]: {item['metrics']}" + f"\n[LLM Judge Reasoning]: {eval_record['reasoning']}" + f"\n" + "="*60 + ) + self.logger.info(detailed_info) + return item + + def _update_report(self, data): + """Read existing report, merge new data, and write back""" + report = {} + if os.path.exists(self.report_file): + with open(self.report_file, "r", encoding="utf-8") as f: + try: + report = json.load(f) + except json.JSONDecodeError: + report = {} + report.update(data) + with open(self.report_file, "w", encoding="utf-8") as f: + json.dump(report, f, indent=4, ensure_ascii=False) + self.logger.info(f"Report updated -> {self.report_file}") diff --git a/pyproject.toml b/pyproject.toml index 6aaf2c03..0b6a68f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -146,6 +146,12 @@ bot-opencode = ["opencode-ai>=0.1.0a0"] bot-full = [ "openviking[bot,bot-langfuse,bot-telegram,bot-feishu,bot-dingtalk,bot-slack,bot-qq,bot-sandbox,bot-fuse,bot-opencode]", ] +benchmark = [ + "langchain>=1.0.0", + "langchain-core>=1.0.0", + "langchain-openai>=1.0.0", + "tiktoken>=0.5.0", +] [project.urls] Homepage = "https://github.com/volcengine/openviking" diff --git a/uv.lock b/uv.lock index 59e6638e..c4afdd86 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.10" resolution-markers = [ "python_full_version >= '3.14' and sys_platform == 'win32'", @@ -1454,7 +1454,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/38/3f/9859f655d11901e7b2996c6e3d33e0caa9a1d4572c3bc61ed0faa64b2f4c/greenlet-3.3.2-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:9bc885b89709d901859cf95179ec9f6bb67a3d2bb1f0e88456461bd4b7f8fd0d", size = 277747, upload-time = "2026-02-20T20:16:21.325Z" }, { url = "https://files.pythonhosted.org/packages/fb/07/cb284a8b5c6498dbd7cba35d31380bb123d7dceaa7907f606c8ff5993cbf/greenlet-3.3.2-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b568183cf65b94919be4438dc28416b234b678c608cafac8874dfeeb2a9bbe13", size = 579202, upload-time = "2026-02-20T20:47:28.955Z" }, { url = "https://files.pythonhosted.org/packages/ed/45/67922992b3a152f726163b19f890a85129a992f39607a2a53155de3448b8/greenlet-3.3.2-cp310-cp310-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:527fec58dc9f90efd594b9b700662ed3fb2493c2122067ac9c740d98080a620e", size = 590620, upload-time = "2026-02-20T20:55:55.581Z" }, - { url = "https://files.pythonhosted.org/packages/03/5f/6e2a7d80c353587751ef3d44bb947f0565ec008a2e0927821c007e96d3a7/greenlet-3.3.2-cp310-cp310-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:508c7f01f1791fbc8e011bd508f6794cb95397fdb198a46cb6635eb5b78d85a7", size = 602132, upload-time = "2026-02-20T21:02:43.261Z" }, { url = "https://files.pythonhosted.org/packages/ad/55/9f1ebb5a825215fadcc0f7d5073f6e79e3007e3282b14b22d6aba7ca6cb8/greenlet-3.3.2-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ad0c8917dd42a819fe77e6bdfcb84e3379c0de956469301d9fd36427a1ca501f", size = 591729, upload-time = "2026-02-20T20:20:58.395Z" }, { url = "https://files.pythonhosted.org/packages/24/b4/21f5455773d37f94b866eb3cf5caed88d6cea6dd2c6e1f9c34f463cba3ec/greenlet-3.3.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:97245cc10e5515dbc8c3104b2928f7f02b6813002770cfaffaf9a6e0fc2b94ef", size = 1551946, upload-time = "2026-02-20T20:49:31.102Z" }, { url = "https://files.pythonhosted.org/packages/00/68/91f061a926abead128fe1a87f0b453ccf07368666bd59ffa46016627a930/greenlet-3.3.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8c1fdd7d1b309ff0da81d60a9688a8bd044ac4e18b250320a96fc68d31c209ca", size = 1618494, upload-time = "2026-02-20T20:21:06.541Z" }, @@ -1462,7 +1461,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f3/47/16400cb42d18d7a6bb46f0626852c1718612e35dcb0dffa16bbaffdf5dd2/greenlet-3.3.2-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:c56692189a7d1c7606cb794be0a8381470d95c57ce5be03fb3d0ef57c7853b86", size = 278890, upload-time = "2026-02-20T20:19:39.263Z" }, { url = "https://files.pythonhosted.org/packages/a3/90/42762b77a5b6aa96cd8c0e80612663d39211e8ae8a6cd47c7f1249a66262/greenlet-3.3.2-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ebd458fa8285960f382841da585e02201b53a5ec2bac6b156fc623b5ce4499f", size = 581120, upload-time = "2026-02-20T20:47:30.161Z" }, { url = "https://files.pythonhosted.org/packages/bf/6f/f3d64f4fa0a9c7b5c5b3c810ff1df614540d5aa7d519261b53fba55d4df9/greenlet-3.3.2-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a443358b33c4ec7b05b79a7c8b466f5d275025e750298be7340f8fc63dff2a55", size = 594363, upload-time = "2026-02-20T20:55:56.965Z" }, - { url = "https://files.pythonhosted.org/packages/9c/8b/1430a04657735a3f23116c2e0d5eb10220928846e4537a938a41b350bed6/greenlet-3.3.2-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4375a58e49522698d3e70cc0b801c19433021b5c37686f7ce9c65b0d5c8677d2", size = 605046, upload-time = "2026-02-20T21:02:45.234Z" }, { url = "https://files.pythonhosted.org/packages/72/83/3e06a52aca8128bdd4dcd67e932b809e76a96ab8c232a8b025b2850264c5/greenlet-3.3.2-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e2cd90d413acbf5e77ae41e5d3c9b3ac1d011a756d7284d7f3f2b806bbd6358", size = 594156, upload-time = "2026-02-20T20:20:59.955Z" }, { url = "https://files.pythonhosted.org/packages/70/79/0de5e62b873e08fe3cef7dbe84e5c4bc0e8ed0c7ff131bccb8405cd107c8/greenlet-3.3.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:442b6057453c8cb29b4fb36a2ac689382fc71112273726e2423f7f17dc73bf99", size = 1554649, upload-time = "2026-02-20T20:49:32.293Z" }, { url = "https://files.pythonhosted.org/packages/5a/00/32d30dee8389dc36d42170a9c66217757289e2afb0de59a3565260f38373/greenlet-3.3.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:45abe8eb6339518180d5a7fa47fa01945414d7cca5ecb745346fc6a87d2750be", size = 1619472, upload-time = "2026-02-20T20:21:07.966Z" }, @@ -1471,7 +1469,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ea/ab/1608e5a7578e62113506740b88066bf09888322a311cff602105e619bd87/greenlet-3.3.2-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:ac8d61d4343b799d1e526db579833d72f23759c71e07181c2d2944e429eb09cd", size = 280358, upload-time = "2026-02-20T20:17:43.971Z" }, { url = "https://files.pythonhosted.org/packages/a5/23/0eae412a4ade4e6623ff7626e38998cb9b11e9ff1ebacaa021e4e108ec15/greenlet-3.3.2-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ceec72030dae6ac0c8ed7591b96b70410a8be370b6a477b1dbc072856ad02bd", size = 601217, upload-time = "2026-02-20T20:47:31.462Z" }, { url = "https://files.pythonhosted.org/packages/f8/16/5b1678a9c07098ecb9ab2dd159fafaf12e963293e61ee8d10ecb55273e5e/greenlet-3.3.2-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a2a5be83a45ce6188c045bcc44b0ee037d6a518978de9a5d97438548b953a1ac", size = 611792, upload-time = "2026-02-20T20:55:58.423Z" }, - { url = "https://files.pythonhosted.org/packages/5c/c5/cc09412a29e43406eba18d61c70baa936e299bc27e074e2be3806ed29098/greenlet-3.3.2-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ae9e21c84035c490506c17002f5c8ab25f980205c3e61ddb3a2a2a2e6c411fcb", size = 626250, upload-time = "2026-02-20T21:02:46.596Z" }, { url = "https://files.pythonhosted.org/packages/50/1f/5155f55bd71cabd03765a4aac9ac446be129895271f73872c36ebd4b04b6/greenlet-3.3.2-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:43e99d1749147ac21dde49b99c9abffcbc1e2d55c67501465ef0930d6e78e070", size = 613875, upload-time = "2026-02-20T20:21:01.102Z" }, { url = "https://files.pythonhosted.org/packages/fc/dd/845f249c3fcd69e32df80cdab059b4be8b766ef5830a3d0aa9d6cad55beb/greenlet-3.3.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4c956a19350e2c37f2c48b336a3afb4bff120b36076d9d7fb68cb44e05d95b79", size = 1571467, upload-time = "2026-02-20T20:49:33.495Z" }, { url = "https://files.pythonhosted.org/packages/2a/50/2649fe21fcc2b56659a452868e695634722a6655ba245d9f77f5656010bf/greenlet-3.3.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6c6f8ba97d17a1e7d664151284cb3315fc5f8353e75221ed4324f84eb162b395", size = 1640001, upload-time = "2026-02-20T20:21:09.154Z" }, @@ -1480,7 +1477,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ac/48/f8b875fa7dea7dd9b33245e37f065af59df6a25af2f9561efa8d822fde51/greenlet-3.3.2-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:aa6ac98bdfd716a749b84d4034486863fd81c3abde9aa3cf8eff9127981a4ae4", size = 279120, upload-time = "2026-02-20T20:19:01.9Z" }, { url = "https://files.pythonhosted.org/packages/49/8d/9771d03e7a8b1ee456511961e1b97a6d77ae1dea4a34a5b98eee706689d3/greenlet-3.3.2-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ab0c7e7901a00bc0a7284907273dc165b32e0d109a6713babd04471327ff7986", size = 603238, upload-time = "2026-02-20T20:47:32.873Z" }, { url = "https://files.pythonhosted.org/packages/59/0e/4223c2bbb63cd5c97f28ffb2a8aee71bdfb30b323c35d409450f51b91e3e/greenlet-3.3.2-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d248d8c23c67d2291ffd47af766e2a3aa9fa1c6703155c099feb11f526c63a92", size = 614219, upload-time = "2026-02-20T20:55:59.817Z" }, - { url = "https://files.pythonhosted.org/packages/94/2b/4d012a69759ac9d77210b8bfb128bc621125f5b20fc398bce3940d036b1c/greenlet-3.3.2-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ccd21bb86944ca9be6d967cf7691e658e43417782bce90b5d2faeda0ff78a7dd", size = 628268, upload-time = "2026-02-20T21:02:48.024Z" }, { url = "https://files.pythonhosted.org/packages/7a/34/259b28ea7a2a0c904b11cd36c79b8cef8019b26ee5dbe24e73b469dea347/greenlet-3.3.2-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b6997d360a4e6a4e936c0f9625b1c20416b8a0ea18a8e19cabbefc712e7397ab", size = 616774, upload-time = "2026-02-20T20:21:02.454Z" }, { url = "https://files.pythonhosted.org/packages/0a/03/996c2d1689d486a6e199cb0f1cf9e4aa940c500e01bdf201299d7d61fa69/greenlet-3.3.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:64970c33a50551c7c50491671265d8954046cb6e8e2999aacdd60e439b70418a", size = 1571277, upload-time = "2026-02-20T20:49:34.795Z" }, { url = "https://files.pythonhosted.org/packages/d9/c4/2570fc07f34a39f2caf0bf9f24b0a1a0a47bc2e8e465b2c2424821389dfc/greenlet-3.3.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1a9172f5bf6bd88e6ba5a84e0a68afeac9dc7b6b412b245dd64f52d83c81e55b", size = 1640455, upload-time = "2026-02-20T20:21:10.261Z" }, @@ -1489,7 +1485,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3f/ae/8bffcbd373b57a5992cd077cbe8858fff39110480a9d50697091faea6f39/greenlet-3.3.2-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:8d1658d7291f9859beed69a776c10822a0a799bc4bfe1bd4272bb60e62507dab", size = 279650, upload-time = "2026-02-20T20:18:00.783Z" }, { url = "https://files.pythonhosted.org/packages/d1/c0/45f93f348fa49abf32ac8439938726c480bd96b2a3c6f4d949ec0124b69f/greenlet-3.3.2-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:18cb1b7337bca281915b3c5d5ae19f4e76d35e1df80f4ad3c1a7be91fadf1082", size = 650295, upload-time = "2026-02-20T20:47:34.036Z" }, { url = "https://files.pythonhosted.org/packages/b3/de/dd7589b3f2b8372069ab3e4763ea5329940fc7ad9dcd3e272a37516d7c9b/greenlet-3.3.2-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2e47408e8ce1c6f1ceea0dffcdf6ebb85cc09e55c7af407c99f1112016e45e9", size = 662163, upload-time = "2026-02-20T20:56:01.295Z" }, - { url = "https://files.pythonhosted.org/packages/cd/ac/85804f74f1ccea31ba518dcc8ee6f14c79f73fe36fa1beba38930806df09/greenlet-3.3.2-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e3cb43ce200f59483eb82949bf1835a99cf43d7571e900d7c8d5c62cdf25d2f9", size = 675371, upload-time = "2026-02-20T21:02:49.664Z" }, { url = "https://files.pythonhosted.org/packages/d2/d8/09bfa816572a4d83bccd6750df1926f79158b1c36c5f73786e26dbe4ee38/greenlet-3.3.2-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:63d10328839d1973e5ba35e98cccbca71b232b14051fd957b6f8b6e8e80d0506", size = 664160, upload-time = "2026-02-20T20:21:04.015Z" }, { url = "https://files.pythonhosted.org/packages/48/cf/56832f0c8255d27f6c35d41b5ec91168d74ec721d85f01a12131eec6b93c/greenlet-3.3.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8e4ab3cfb02993c8cc248ea73d7dae6cec0253e9afa311c9b37e603ca9fad2ce", size = 1619181, upload-time = "2026-02-20T20:49:36.052Z" }, { url = "https://files.pythonhosted.org/packages/0a/23/b90b60a4aabb4cec0796e55f25ffbfb579a907c3898cd2905c8918acaa16/greenlet-3.3.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:94ad81f0fd3c0c0681a018a976e5c2bd2ca2d9d94895f23e7bb1af4e8af4e2d5", size = 1687713, upload-time = "2026-02-20T20:21:11.684Z" }, @@ -1498,7 +1493,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/98/6d/8f2ef704e614bcf58ed43cfb8d87afa1c285e98194ab2cfad351bf04f81e/greenlet-3.3.2-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:e26e72bec7ab387ac80caa7496e0f908ff954f31065b0ffc1f8ecb1338b11b54", size = 286617, upload-time = "2026-02-20T20:19:29.856Z" }, { url = "https://files.pythonhosted.org/packages/5e/0d/93894161d307c6ea237a43988f27eba0947b360b99ac5239ad3fe09f0b47/greenlet-3.3.2-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b466dff7a4ffda6ca975979bab80bdadde979e29fc947ac3be4451428d8b0e4", size = 655189, upload-time = "2026-02-20T20:47:35.742Z" }, { url = "https://files.pythonhosted.org/packages/f5/2c/d2d506ebd8abcb57386ec4f7ba20f4030cbe56eae541bc6fd6ef399c0b41/greenlet-3.3.2-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b8bddc5b73c9720bea487b3bffdb1840fe4e3656fba3bd40aa1489e9f37877ff", size = 658225, upload-time = "2026-02-20T20:56:02.527Z" }, - { url = "https://files.pythonhosted.org/packages/d1/67/8197b7e7e602150938049d8e7f30de1660cfb87e4c8ee349b42b67bdb2e1/greenlet-3.3.2-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:59b3e2c40f6706b05a9cd299c836c6aa2378cabe25d021acd80f13abf81181cf", size = 666581, upload-time = "2026-02-20T21:02:51.526Z" }, { url = "https://files.pythonhosted.org/packages/8e/30/3a09155fbf728673a1dea713572d2d31159f824a37c22da82127056c44e4/greenlet-3.3.2-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b26b0f4428b871a751968285a1ac9648944cea09807177ac639b030bddebcea4", size = 657907, upload-time = "2026-02-20T20:21:05.259Z" }, { url = "https://files.pythonhosted.org/packages/f3/fd/d05a4b7acd0154ed758797f0a43b4c0962a843bedfe980115e842c5b2d08/greenlet-3.3.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1fb39a11ee2e4d94be9a76671482be9398560955c9e568550de0224e41104727", size = 1618857, upload-time = "2026-02-20T20:49:37.309Z" }, { url = "https://files.pythonhosted.org/packages/6f/e1/50ee92a5db521de8f35075b5eff060dd43d39ebd46c2181a2042f7070385/greenlet-3.3.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:20154044d9085151bc309e7689d6f7ba10027f8f5a8c0676ad398b951913d89e", size = 1680010, upload-time = "2026-02-20T20:21:13.427Z" }, @@ -3286,6 +3280,12 @@ dependencies = [ ] [package.optional-dependencies] +benchmark = [ + { name = "langchain" }, + { name = "langchain-core" }, + { name = "langchain-openai" }, + { name = "tiktoken" }, +] bot = [ { name = "beautifulsoup4" }, { name = "croniter" }, @@ -3430,6 +3430,9 @@ requires-dist = [ { name = "httpx", extras = ["socks"], marker = "extra == 'bot'", specifier = ">=0.25.0" }, { name = "jinja2", specifier = ">=3.1.6" }, { name = "json-repair", specifier = ">=0.25.0" }, + { name = "langchain", marker = "extra == 'benchmark'", specifier = ">=1.0.0" }, + { name = "langchain-core", marker = "extra == 'benchmark'", specifier = ">=1.0.0" }, + { name = "langchain-openai", marker = "extra == 'benchmark'", specifier = ">=1.0.0" }, { name = "langfuse", marker = "extra == 'bot-langfuse'", specifier = ">=3.0.0" }, { name = "lark-oapi", marker = "extra == 'bot-feishu'", specifier = ">=1.0.0" }, { name = "litellm", specifier = ">=1.0.0" }, @@ -3481,6 +3484,7 @@ requires-dist = [ { name = "sphinx", marker = "extra == 'doc'", specifier = ">=7.0.0" }, { name = "sphinx-rtd-theme", marker = "extra == 'doc'", specifier = ">=1.3.0" }, { name = "tabulate", specifier = ">=0.9.0" }, + { name = "tiktoken", marker = "extra == 'benchmark'", specifier = ">=0.5.0" }, { name = "tree-sitter", specifier = ">=0.23.0" }, { name = "tree-sitter-c-sharp", specifier = ">=0.23.0" }, { name = "tree-sitter-cpp", specifier = ">=0.23.0" }, @@ -3502,7 +3506,7 @@ requires-dist = [ { name = "xlrd", specifier = ">=2.0.1" }, { name = "xxhash", specifier = ">=3.0.0" }, ] -provides-extras = ["test", "dev", "doc", "eval", "build", "bot", "bot-langfuse", "bot-telegram", "bot-feishu", "bot-dingtalk", "bot-slack", "bot-qq", "bot-sandbox", "bot-fuse", "bot-opencode", "bot-full"] +provides-extras = ["test", "dev", "doc", "eval", "build", "bot", "bot-langfuse", "bot-telegram", "bot-feishu", "bot-dingtalk", "bot-slack", "bot-qq", "bot-sandbox", "bot-fuse", "bot-opencode", "bot-full", "benchmark"] [package.metadata.requires-dev] dev = [{ name = "pytest", specifier = ">=9.0.2" }]