llm-blender/Unified-Feedback

license: mit
configs:

• config_name: Nectar
  data_files:
  • split: train
    path: Nectar/train-*
  • split: val
    path: Nectar/val-*
• config_name: UltraFeedback
  data_files:
  • split: train
    path: UltraFeedback/train-*
  • split: val
    path: UltraFeedback/val-*
• config_name: all
  data_files:
  • split: train
    path: all/train-*
  • split: val
    path: all/val-*
• config_name: chatbot_arena_conversations
  data_files:
  • split: train
    path: chatbot_arena_conversations/train-*
  • split: val
    path: chatbot_arena_conversations/val-*
• config_name: hh-rlhf
  data_files:
  • split: train
    path: hh-rlhf/train-*
  • split: val
    path: hh-rlhf/val-*
• config_name: summarize_from_feedback
  data_files:
  • split: train
    path: summarize_from_feedback/train-*
  • split: val
    path: summarize_from_feedback/val-*
• config_name: synthetic-instruct-gptj-pairwise
  data_files:
  • split: train
    path: synthetic-instruct-gptj-pairwise/train-*
  • split: val
    path: synthetic-instruct-gptj-pairwise/val-*
• config_name: ultrafeedback-binarized-preferences-cleaned
  data_files:
  • split: train
    path: ultrafeedback-binarized-preferences-cleaned/train-*
  • split: val
    path: ultrafeedback-binarized-preferences-cleaned/val-*
• config_name: webgpt_comparisons
  data_files:
  • split: train
    path: webgpt_comparisons/train-*
  • split: val
    path: webgpt_comparisons/val-*
    dataset_info:
• config_name: Nectar
  features:
  • name: id
    dtype: string
  • name: conv_A
    list:
    • name: role
      dtype: string
    • name: content
      dtype: string
  • name: conv_B
    list:
    • name: role
      dtype: string
    • name: content
      dtype: string
  • name: conv_A_rating
    dtype: float32
  • name: conv_B_rating
    dtype: float32
  • name: num_turns
    dtype: int32
  • name: source
    dtype: string
    splits:
  • name: train
    num_bytes: 1081068032
    num_examples: 364908
  • name: val
    num_bytes: 3078273
    num_examples: 1000
    download_size: 1192518861
    dataset_size: 1084146305
• config_name: UltraFeedback
  features:
  • name: id
    dtype: string
  • name: conv_A
    list:
    • name: role
      dtype: string
    • name: content
      dtype: string
  • name: conv_B
    list:
    • name: role
      dtype: string
    • name: content
      dtype: string
  • name: conv_A_rating
    dtype: float32
  • name: conv_B_rating
    dtype: float32
  • name: num_turns
    dtype: int32
  • name: source
    dtype: string
    splits:
  • name: train
    num_bytes: 482967411
    num_examples: 126932
  • name: val
    num_bytes: 3796982
    num_examples: 1000
    download_size: 355823052
    dataset_size: 486764393
• config_name: all
  features:
  • name: id
    dtype: string
  • name: conv_A
    list:
    • name: role
      dtype: string
    • name: content
      dtype: string
  • name: conv_B
    list:
    • name: role
      dtype: string
    • name: content
      dtype: string
  • name: conv_A_rating
    dtype: float32
  • name: conv_B_rating
    dtype: float32
  • name: num_turns
    dtype: int32
  • name: source
    dtype: string
    splits:
  • name: train
    num_bytes: 2539681817
    num_examples: 884515
  • name: val
    num_bytes: 21148253
    num_examples: 8000
    download_size: 1968174668
    dataset_size: 2560830070
• config_name: chatbot_arena_conversations
  features:
  • name: id
    dtype: string
  • name: conv_A
    list:
    • name: role
      dtype: string
    • name: content
      dtype: string
  • name: conv_B
    list:
    • name: role
      dtype: string
    • name: content
      dtype: string
  • name: conv_A_rating
    dtype: float32
  • name: conv_B_rating
    dtype: float32
  • name: num_turns
    dtype: int32
  • name: source
    dtype: string
    splits:
  • name: train
    num_bytes: 74939906
    num_examples: 32000
  • name: val
    num_bytes: 2316679
    num_examples: 1000
    download_size: 298130885
    dataset_size: 77256585
• config_name: hh-rlhf
  features:
  • name: id
    dtype: string
  • name: conv_A
    list:
    • name: role
      dtype: string
    • name: content
      dtype: string
  • name: conv_B
    list:
    • name: role
      dtype: string
    • name: content
      dtype: string
  • name: conv_A_rating
    dtype: float32
  • name: conv_B_rating
    dtype: float32
  • name: num_turns
    dtype: int32
  • name: source
    dtype: string
    splits:
  • name: train
    num_bytes: 317595099
    num_examples: 158179
  • name: val
    num_bytes: 1982489
    num_examples: 1000
    download_size: 699435351
    dataset_size: 319577588
• config_name: summarize_from_feedback
  features:
  • name: id
    dtype: string
  • name: conv_A
    list:
    • name: role
      dtype: string
    • name: content
      dtype: string
  • name: conv_B
    list:
    • name: role
      dtype: string
    • name: content
      dtype: string
  • name: conv_A_rating
    dtype: float32
  • name: conv_B_rating
    dtype: float32
  • name: num_turns
    dtype: int32
  • name: source
    dtype: string
    splits:
  • name: train
    num_bytes: 274935253
    num_examples: 91858
  • name: val
    num_bytes: 2971753
    num_examples: 1000
    download_size: 98251164
    dataset_size: 277907006
• config_name: synthetic-instruct-gptj-pairwise
  features:
  • name: id
    dtype: string
  • name: conv_A
    list:
    • name: role
      dtype: string
    • name: content
      dtype: string
  • name: conv_B
    list:
    • name: role
      dtype: string
    • name: content
      dtype: string
  • name: conv_A_rating
    dtype: float32
  • name: conv_B_rating
    dtype: float32
  • name: num_turns
    dtype: int32
  • name: source
    dtype: string
    splits:
  • name: train
    num_bytes: 37011887
    num_examples: 32143
  • name: val
    num_bytes: 1141460
    num_examples: 1000
    download_size: 79160756
    dataset_size: 38153347
• config_name: ultrafeedback-binarized-preferences-cleaned
  features:
  • name: id
    dtype: string
  • name: conv_A
    list:
    • name: role
      dtype: string
    • name: content
      dtype: string
  • name: conv_B
    list:
    • name: role
      dtype: string
    • name: content
      dtype: string
  • name: conv_A_rating
    dtype: float32
  • name: conv_B_rating
    dtype: float32
  • name: num_turns
    dtype: int32
  • name: source
    dtype: string
    splits:
  • name: train
    num_bytes: 242561699
    num_examples: 59917
  • name: val
    num_bytes: 4009990
    num_examples: 1000
    download_size: 366415038
    dataset_size: 246571689
• config_name: webgpt
  features:
  • name: id
    dtype: string
  • name: conv_A
    list:
    • name: role
      dtype: string
    • name: content
      dtype: string
  • name: conv_B
    list:
    • name: role
      dtype: string
    • name: content
      dtype: string
  • name: conv_A_rating
    dtype: float32
  • name: conv_B_rating
    dtype: float32
  • name: num_turns
    dtype: int32
  • name: source
    dtype: string
    splits:
  • name: train
    num_bytes: 37161667
    num_examples: 18578
  • name: val
    num_bytes: 1990672
    num_examples: 1000
    download_size: 22652378
    dataset_size: 39152339
• config_name: webgpt_comparisons
  features:
  • name: id
    dtype: string
  • name: conv_A
    list:
    • name: role
      dtype: string
    • name: content
      dtype: string
  • name: conv_B
    list:
    • name: role
      dtype: string
    • name: content
      dtype: string
  • name: conv_A_rating
    dtype: float32
  • name: conv_B_rating
    dtype: float32
  • name: num_turns
    dtype: int32
  • name: source
    dtype: string
    splits:
  • name: train
    num_bytes: 37384603
    num_examples: 18578
  • name: val
    num_bytes: 2002672
    num_examples: 1000
    download_size: 67963041
    dataset_size: 39387275

Collections of pairwise feedback datasets.

• openai/summarize_from_feedback
• openai/webgpt_comparisons
• Dahoas/instruct-synthetic-prompt-responses
• Anthropic/hh-rlhf
• lmsys/chatbot_arena_conversations
• openbmb/UltraFeedback
• argilla/ultrafeedback-binarized-preferences-cleaned
• berkeley-nest/Nectar

Codes to reproduce the dataset: jdf-prog/UnifiedFeedback

Dataset formats

{
  "id": "...",
  "conv_A": [
    {
      "role": "user",
      "content": "...",
    },
    {
      "role": "assistant",
      "content": "...",
    },
    ...
  ]
  "conv_B": [
    {
      "role": "user",
      "content": "...",
    },
    {
      "role": "assistant",
      "content": "...",
    },
    ...
  ]
  "conv_A_rating": 1, # not necessarily always 1 and 0. The general principle is that higher is better
  "conv_B_rating": 0,  # not necessarily always 1 and 0. The general principle is that higher is better
  "num_turns": 1, # the number of conversation turns. one user and one assistant counts for 1 turn.
  "source": "...", # hugging face dataset source
}

To convert to pairwise prompt and response to train reward models like PairRM.

Into the mixinstruct format:

Refer to script to_mixinstruct_format.py to convert the unified feedback into the format that can use llm-blender repo to train the ranker

The following are the functions used in the above script

• For conversations with 1 turn

Simply take the user content and assistant content for 1 turn as the input and output.

def get_pair_from_conv_for_single_turn(convAs: List[str], convBs: List[str]):
    """Compare two conversations by takeing USER turns as inputs and ASSISTANT turns as candidates
        Multi-turn conversations comparison is also supportted.
        a conversation format is:
        ```python
        [
            {
                "content": "hello",
                "role": "USER"
            },
            {
                "content": "hi",
                "role": "ASSISTANT"
            },
            ...
        ]
        ```
    Args:
        convAs (List[List[dict]]): List of conversations
        convAs (List[List[dict]]): List of conversations
    """
    for c in convAs + convBs:
        assert len(c) % 2 == 0, "Each conversation must have even number of turns"
        assert all([c[i]['role'].upper() == 'USER' for i in range(0, len(c), 2)]), "Each even turn must be USER"
        assert all([c[i]['role'].upper() == 'ASSISTANT' for i in range(1, len(c), 2)]), "Each odd turn must be ASSISTANT"
    # check conversations correctness
    assert len(convAs) == len(convBs), "Number of conversations must be the same"
    for c_a, c_b in zip(convAs, convBs):
        assert len(c_a) == len(c_b), "Number of turns in each conversation must be the same"
        assert all([c_a[i]['content'] == c_b[i]['content'] for i in range(0, len(c_a), 2)]), "USER turns must be the same"
    
    inputs = [
        convAs[i][0]['content'] for i in range(len(convAs))
    ]
    cand1_texts = [
        convAs[i][1]['content'] for i in range(len(convAs))
    ]
    cand2_texts = [
        convBs[i][1]['content'] for i in range(len(convBs))
    ]
    return inputs, cand1_texts, cand2_texts

• For conversations with more than 1 turns

convert it into input output pairs using the following function:

def get_pair_from_conv(convAs: List[str], convBs: List[str]):
    """Compare two conversations by takeing USER turns as inputs and ASSISTANT turns as candidates
        Multi-turn conversations comparison is also supportted.
        a conversation format is:
        ```python
        [
            {
                "content": "hello",
                "role": "USER"
            },
            {
                "content": "hi",
                "role": "ASSISTANT"
            },
            ...
        ]
        ```
    Args:
        convAs (List[List[dict]]): List of conversations
        convAs (List[List[dict]]): List of conversations
    """
    for c in convAs + convBs:
        assert len(c) % 2 == 0, "Each conversation must have even number of turns"
        assert all([c[i]['role'].upper() == 'USER' for i in range(0, len(c), 2)]), "Each even turn must be USER"
        assert all([c[i]['role'].upper() == 'ASSISTANT' for i in range(1, len(c), 2)]), "Each odd turn must be ASSISTANT"
    # check conversations correctness
    assert len(convAs) == len(convBs), "Number of conversations must be the same"
    for c_a, c_b in zip(convAs, convBs):
        assert len(c_a) == len(c_b), "Number of turns in each conversation must be the same"
        assert all([c_a[i]['content'] == c_b[i]['content'] for i in range(0, len(c_a), 2)]), "USER turns must be the same"
    
    instructions = ["Finish the following coversation in each i-th turn by filling in <Response i> with your response."] * len(convAs)
    inputs = [
        "\n".join([
            "USER: " + x[i]['content'] +
            f"\nAssistant: <Response {i//2+1}>" for i in range(0, len(x), 2)
        ]) for x in convAs
    ]
    cand1_texts = [
        "\n".join([
            f"<Response {i//2+1}>: " + x[i]['content'] for i in range(1, len(x), 2)
        ]) for x in convAs
    ]
    cand2_texts = [
        "\n".join([
            f"<Response {i//2+1}>: " + x[i]['content'] for i in range(1, len(x), 2)
        ]) for x in convBs
    ]
    inputs = [inst + inp for inst, inp in zip(instructions, inputs)]
    return inputs, cand1_texts, cand2_texts