license: apache-2.0
from datasets import load_dataset, concatenate_datasets
# List of dataset paths
dataset_paths = [
"V3N0M/Jenna-50K-Alpaca-Uncensored",
"SaisExperiments/Alpaca-Uncensored",
"SaisExperiments/Big-Alpaca-Uncensored",
"xzuyn/open-instruct-uncensored-alpaca",
"xzuyn/tulu-uncensored-alpaca",
"xzuyn/tv-alpaca-open-instruct-uncensored-blend",
"dim/dolphin_flan1m_alpaca_uncensored_3k",
"dataautogpt3/flan1m-alpaca-uncensored",
"ShubhVenom/Uncensored-Alpaca-v01",
"V3N0M/Uncensored-Alpaca",
"Xennon-BD/Alpaca-uncensored",
"VinyVan/flanMini-alpaca-uncensored_bambara"
]
# Load the first dataset to get reference columns
dataset1 = load_dataset(dataset_paths[0], split="train")
reference_columns = dataset1.column_names
# Load and select columns for the remaining datasets
datasets = [dataset1]
for path in dataset_paths[1:]:
dataset = load_dataset(path, split="train")
dataset = dataset.select_columns(reference_columns)
datasets.append(dataset)
# Merge all datasets
merged_dataset = concatenate_datasets(datasets)
# Print the number of rows in the merged dataset
print(f"line: {len(merged_dataset)}")
# Save the merged dataset to disk
merged_dataset.save_to_disk("merged_uncensored_alpaca")