annotations_creators:

• machine-generated
  language_creators:
• found
• machine-generated
  language:
• en
  license:
• cc-by-4.0
  multilinguality:
• monolingual
  size_categories:
• 10K<n<100K
  source_datasets:
• extended|cardiffnlp/tweet_eval
  task_categories:
• text-classification
  task_ids:
• acceptability-classification
• fact-checking
  paperswithcode_id: unmasking-the-imposters-in-domain-detection
  pretty_name: "Unmasking the Imposters: Machine-Generated Tweet Detection Dataset"
  tags:
• machine-generated-text
• tweet-detection
• social-media
• llm-detection
• censorship
• content-moderation
• ai-detection
• transformer-models
  config_names:
• dolphin-2.8-mistral-7b-v02
• dolphin-2.9-llama3-8b
• dolphin-2.9.2-qwen2-7b
• gpt4o
• hermes-2-pro-llama-3-8b
• meta-llama-3-8b-instruct
• mistral-7b-instruct-v0.2
• openhermes-2.5-mistral-7b
• qwen2-7b-instruct
  dataset_info:
• config_name: dolphin-2.8-mistral-7b-v02
  features:
  • name: text
    dtype: string
  • name: label
    dtype:
    class_label:
    names:
    '0': human
    '1': generated

  splits:

  • name: train
    num_examples: 7088
  • name: validation
    num_examples: 886
  • name: test
    num_examples: 886
• config_name: dolphin-2.9-llama3-8b
  features:
  • name: text
    dtype: string
  • name: label
    dtype:
    class_label:
    names:
    '0': human
    '1': generated

  splits:

  • name: train
    num_examples: 7088
  • name: validation
    num_examples: 886
  • name: test
    num_examples: 886
• config_name: dolphin-2.9.2-qwen2-7b
  features:
  • name: text
    dtype: string
  • name: label
    dtype:
    class_label:
    names:
    '0': human
    '1': generated

  splits:

  • name: train
    num_examples: 7088
  • name: validation
    num_examples: 886
  • name: test
    num_examples: 886
• config_name: gpt4o
  features:
  • name: text
    dtype: string
  • name: label
    dtype:
    class_label:
    names:
    '0': human
    '1': generated

  splits:

  • name: train
    num_examples: 7088
  • name: validation
    num_examples: 886
  • name: test
    num_examples: 886
• config_name: hermes-2-pro-llama-3-8b
  features:
  • name: text
    dtype: string
  • name: label
    dtype:
    class_label:
    names:
    '0': human
    '1': generated

  splits:

  • name: train
    num_examples: 7088
  • name: validation
    num_examples: 886
  • name: test
    num_examples: 886
• config_name: meta-llama-3-8b-instruct
  features:
  • name: text
    dtype: string
  • name: label
    dtype:
    class_label:
    names:
    '0': human
    '1': generated

  splits:

  • name: train
    num_examples: 7088
  • name: validation
    num_examples: 886
  • name: test
    num_examples: 886
• config_name: mistral-7b-instruct-v0.2
  features:
  • name: text
    dtype: string
  • name: label
    dtype:
    class_label:
    names:
    '0': human
    '1': generated

  splits:

  • name: train
    num_examples: 7088
  • name: validation
    num_examples: 886
  • name: test
    num_examples: 886
• config_name: openhermes-2.5-mistral-7b
  features:
  • name: text
    dtype: string
  • name: label
    dtype:
    class_label:
    names:
    '0': human
    '1': generated

  splits:

  • name: train
    num_examples: 7088
  • name: validation
    num_examples: 886
  • name: test
    num_examples: 886
• config_name: qwen2-7b-instruct
  features:
  • name: text
    dtype: string
  • name: label
    dtype:
    class_label:
    names:
    '0': human
    '1': generated

  splits:

  • name: train
    num_examples: 7088
  • name: validation
    num_examples: 886
  • name: test
    num_examples: 886
    train-eval-index:
• config: meta-llama-3-8b-instruct
  task: text-classification
  task_id: binary_classification
  splits:
  train_split: train
  eval_split: test
  col_mapping:
  text: text
  label: target
  metrics:
  • type: accuracy
    name: Accuracy
  • type: f1
    name: F1 binary
    args:
    average: binary
  • type: precision
    name: Precision binary
    args:
    average: binary
  • type: recall
    name: Recall binary
    args:
    average: binary
  • type: matthews_correlation
    name: Matthews Correlation Coefficient
    configs:
• config_name: dolphin-2.8-mistral-7b-v02
  data_files:
  • split: train
    path: dolphin-2.8-mistral-7b-v02/train.json
  • split: validation
    path: dolphin-2.8-mistral-7b-v02/val.json
  • split: test
    path: dolphin-2.8-mistral-7b-v02/test.json
• config_name: dolphin-2.9-llama3-8b
  data_files:
  • split: train
    path: dolphin-2.9-llama3-8b/train.json
  • split: validation
    path: dolphin-2.9-llama3-8b/val.json
  • split: test
    path: dolphin-2.9-llama3-8b/test.json
• config_name: dolphin-2.9.2-qwen2-7b
  data_files:
  • split: train
    path: dolphin-2.9.2-qwen2-7b/train.json
  • split: validation
    path: dolphin-2.9.2-qwen2-7b/val.json
  • split: test
    path: dolphin-2.9.2-qwen2-7b/test.json
• config_name: gpt4o
  data_files:
  • split: train
    path: gpt4o/train.json
  • split: validation
    path: gpt4o/val.json
  • split: test
    path: gpt4o/test.json
• config_name: hermes-2-pro-llama-3-8b
  data_files:
  • split: train
    path: Hermes-2-Pro-Llama-3-8B/train.json
  • split: validation
    path: Hermes-2-Pro-Llama-3-8B/val.json
  • split: test
    path: Hermes-2-Pro-Llama-3-8B/test.json
• config_name: meta-llama-3-8b-instruct
  data_files:
  • split: train
    path: Meta-Llama-3-8B-Instruct/train.json
  • split: validation
    path: Meta-Llama-3-8B-Instruct/val.json
  • split: test
    path: Meta-Llama-3-8B-Instruct/test.json
• config_name: mistral-7b-instruct-v0.2
  data_files:
  • split: train
    path: Mistral-7B-Instruct-v0.2/train.json
  • split: validation
    path: Mistral-7B-Instruct-v0.2/val.json
  • split: test
    path: Mistral-7B-Instruct-v0.2/test.json
• config_name: openhermes-2.5-mistral-7b
  data_files:
  • split: train
    path: OpenHermes-2.5-Mistral-7B/train.json
  • split: validation
    path: OpenHermes-2.5-Mistral-7B/val.json
  • split: test
    path: OpenHermes-2.5-Mistral-7B/test.json
• config_name: qwen2-7b-instruct
  data_files:
  • split: train
    path: Qwen2-7B-Instruct/train.json
  • split: validation
    path: Qwen2-7B-Instruct/val.json
  • split: test
    path: Qwen2-7B-Instruct/test.json

Unmasking the Imposters: Machine-Generated Tweet Detection Dataset

This dataset contains nine subsets of human and machine-generated tweets designed to evaluate the detection of AI-generated content across censored and uncensored large language models (LLMs). The dataset addresses the gap in understanding how content moderation and domain adaptation affect the detectability of machine-generated text on social media platforms.

• Paper: "Unmasking the Imposters: How Censorship and Domain Adaptation Affect the Detection of Machine-Generated Tweets"

Dataset Details

Dataset Description

This dataset comprises nine distinct subsets, each containing pairs of human-written tweets and machine-generated tweets from different LLM variants. The machine-generated tweets were created using four prominent LLM families (Llama 3, Mistral, Qwen2, and GPT-4o) in both censored and uncensored configurations, fine-tuned on Twitter data from the TweetEval benchmark.

The dataset enables researchers to:

• Study the effectiveness of machine-generated text detection methods

• Analyze how content moderation (censorship) affects text generation quality and detectability

• Evaluate detection performance across different model architectures and sizes

• Investigate the trade-offs between safety constraints and linguistic authenticity

• Curated by: Bryan E. Tuck and Rakesh M. Verma (University of Houston)

• Language(s): English

• License: CC-BY-4.0

Content Warning

Language and Content Warning: This dataset contains real Twitter data and machine-generated text that may include:

• Offensive language, profanity, and hate speech
• Controversial opinions and sensitive topics
• Potentially harmful or toxic content
• Biased or discriminatory language

The uncensored model variants in particular generate content with higher toxicity levels approaching those found in human social media posts. Researchers should exercise appropriate caution when working with this data and consider content filtering based on their specific use case and institutional guidelines.

Dataset Sources

• Base Data: TweetEval benchmark (Barbieri et al., 2020)

Uses

Direct Use

This dataset is intended for:

• Machine-generated text detection research: Training and evaluating detection models
• Content moderation studies: Understanding the impact of censorship on text generation
• Social media analysis: Studying AI-generated content in Twitter-like environments
• Adversarial ML research: Developing robust detection methods against sophisticated generators
• Benchmarking: Comparing detection algorithms across different model types

Out-of-Scope Use

This dataset should not be used for:

• Training models to generate misleading or harmful content
• Developing tools for malicious social media manipulation
• Creating deepfake detection systems without considering ethical implications
• Applications outside the Twitter/social media domain without proper validation
• Fine-tuning models to evade detection systems maliciously

Dataset Structure

Each of the nine dataset subsets contains 8,860 samples with the following structure:

Data Fields

• text: Tweet content (string) - contains both human-written and machine-generated tweets
• label: Binary classification label (integer)
  • 0: Human-written tweet
  • 1: Machine-generated tweet

Important Note: The human tweets (entries with label=0) are identical across all nine dataset subsets, ensuring fair comparison between different LLM variants. Only the machine-generated tweets (entries with label=1) vary between subsets, containing outputs from different censored and uncensored models.

Data Splits

Each subset follows a consistent split:

Dataset Subsets

The nine subsets correspond to different LLM variants:

Censored Models (4 subsets):

• Meta-Llama-3-8B-Instruct (LL3) - 8B parameters
• Mistral-7B-Instruct-v0.2 (Mistral) - 7B parameters
• Qwen2-7B-Instruct (Qwen2) - 7B parameters
• GPT-4o (GPT4o) - Closed-source

Uncensored Models (5 subsets):

• Dolphin-2.9-Llama3-8B (LL3-Dolphin) - 8B parameters
• Hermes-2-Pro-Llama-3-8B (LL3-Hermes) - 8B parameters
• Dolphin-2.8-Mistral-7B-v02 (Mistral-Dolphin) - 7B parameters
• OpenHermes-2.5-Mistral-7B (Mistral-Hermes) - 7B parameters
• Dolphin-2.9.2-Qwen2-7B (Qwen2-Dolphin) - 7B parameters

Evaluation Metrics

The paper evaluates generated text across multiple dimensions:

• Semantic Quality: BERTScore for semantic alignment with human references
• Lexical Richness: Vocabulary size and Moving Average Type-Token Ratio (MTTR)
• Structural Patterns: N-gram diversity, entropy, and intra-sample similarity
• Content Characteristics: Fine-grained toxicity analysis using ToxicBERT
• Detection Performance: Precision, Recall, F1-score, Accuracy, and Matthews Correlation Coefficient (MCC)

Key Research Findings

The dataset enables analysis of several findings:

1. Uncensored Models vs. Detection: Removing content moderation significantly undermines detection effectiveness, with F1 scores dropping from >0.93 for censored models to 0.76-0.89 for uncensored variants
2. Human-like Text Generation: Uncensored models produce text with lexical richness and structural patterns closer to human writing
3. Toxicity Trade-offs: While uncensored models generate more human-like text, they also exhibit higher toxicity levels approaching human baselines
4. Ensemble Superiority: Soft ensemble methods consistently outperform single-model approaches across all datasets
5. Limited Stylometric Value: Traditional stylometric features provide minimal improvement when combined with strong transformer baselines

Dataset Creation

Curation Rationale

This dataset was created to address gaps in machine-generated text detection research:

1. Most existing work focuses on general-purpose datasets rather than domain-specific social media text
2. Limited evaluation of smaller, open-source models that are increasingly used by bad actors
3. Insufficient understanding of how content moderation affects both text quality and detectability
4. Need for comprehensive benchmarks across multiple model architectures and censorship levels

Source Data

Data Collection and Processing

Base Data: Human tweets were sourced from the emotion recognition subset of TweetEval benchmark. Additional TweetEval tasks (hate speech detection, irony detection, offensive language identification, and sentiment analysis) were used only for fine-tuning the models to adapt them to Twitter domain.

Generation Process:

1. Domain Adaptation: All LLMs were fine-tuned on 96,225 tweets from TweetEval using QLoRA (4-bit Quantized Low-Rank Adaptation)
2. Tweet Generation: Used the emotion recognition subset of TweetEval as prompts. Structured prompts instructed models to generate tweets conveying the same emotion as the original while using creative linguistic variations
3. Post-processing: Extensive filtering removed low-quality outputs, unintended phrases, AI-related hashtags, and duplicates

Quality Control:

• Rejection rates varied by model (0.14% for GPT-4o to 12.31% for Qwen2)
• Uncensored models generally had lower rejection rates than censored counterparts
• Strict train/validation/test splits maintained to prevent data leakage

Who are the source data producers?

• Original Human Tweets: Twitter users whose content was included in the TweetEval benchmark
• Generated Content: Nine different LLM variants fine-tuned on social media data
• Dataset Curation: Academic researchers at University of Houston

Annotations

Annotation process

No manual annotation was required as the generation process inherently provides ground truth labels.

Who are the annotators?

Automatic labeling based on data provenance - no human annotators were used.

Personal and Sensitive Information

The dataset is based on publicly available Twitter data from the TweetEval benchmark. While efforts were made to anonymize user mentions and URLs during preprocessing, researchers should be aware that:

• Original tweets may contain opinions, sentiments, or demographic indicators
• Generated tweets may reflect biases present in the training data
• The toxicity analysis in the paper shows varying levels of potentially harmful content across model types

Limitations

Key Limitations

1. Domain Specificity: Results are specific to Twitter-style content and may not generalize to other platforms or text types
2. Language Coverage: Dataset is English-only, limiting multilingual applicability
3. Model Scale: Focus on 7B-8B parameter models may not reflect behavior of larger or smaller models
4. Temporal Constraints: Based on static TweetEval data that may not capture evolving social media language

Citation

@inproceedings{tuck-verma-2025-unmasking,
    title = "Unmasking the Imposters: How Censorship and Domain Adaptation Affect the Detection of Machine-Generated Tweets",
    author = "Tuck, Bryan E.  and Verma, Rakesh",
    booktitle = "Proceedings of the 31st International Conference on Computational Linguistics",
    month = jan,
    year = "2025",
    address = "Abu Dhabi, UAE",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2025.coling-main.607/",
    pages = "9044--9061"
}

@inproceedings{barbieri2020tweeteval,
    title = {{TweetEval:Unified Benchmark and Comparative Evaluation for Tweet Classification}},
    author = {Barbieri, Francesco and Camacho-Collados, Jose and Espinosa-Anke, Luis and Neves, Leonardo},
    booktitle = {Proceedings of Findings of EMNLP},
    year = {2020}
}

Dataset Card Authors

Bryan E. Tuck (University of Houston)

Dataset Card Contact

• Bryan E. Tuck: [email protected]
• Rakesh M. Verma: [email protected]