Thanks for the great work. I encountered the following error when trying loading the dataset from HF, you can reproduce by simply running
Seems that this error is caused by the inconsistent column names in "metadata/docvqa_cot_train.jsonl", should be an easy fix.
---------------------------------------------------------------------------
CastError Traceback (most recent call last)
File ~/fm-efs/znliu/miniconda3/lib/python3.13/site-packages/datasets/builder.py:1871, in ArrowBasedBuilder._prepare_split_single(self, gen_kwargs, fpath, file_format, max_shard_size, job_id)
1870 try:
-> 1871 writer.write_table(table)
1872 except CastError as cast_error:
File ~/fm-efs/znliu/miniconda3/lib/python3.13/site-packages/datasets/arrow_writer.py:643, in ArrowWriter.write_table(self, pa_table, writer_batch_size)
642 pa_table = pa_table.combine_chunks()
--> 643 pa_table = table_cast(pa_table, self._schema)
644 if self.embed_local_files:
File ~/fm-efs/znliu/miniconda3/lib/python3.13/site-packages/datasets/table.py:2293, in table_cast(table, schema)
2292 if table.schema != schema:
-> 2293 return cast_table_to_schema(table, schema)
2294 elif table.schema.metadata != schema.metadata:
File ~/fm-efs/znliu/miniconda3/lib/python3.13/site-packages/datasets/table.py:2241, in cast_table_to_schema(table, schema)
2240 if not table_column_names <= set(schema.names):
-> 2241 raise CastError(
2242 f"Couldn't cast\n{_short_str(table.schema)}\nto\n{_short_str(features)}\nbecause column names don't match",
2243 table_column_names=table.column_names,
2244 requested_column_names=list(features),
2245 )
2246 arrays = [
2247 cast_array_to_feature(
2248 table[name] if name in table_column_names else pa.array([None] * len(table), type=schema.field(name).type),
(...) 2251 for name, feature in features.items()
2252 ]
CastError: Couldn't cast
question: string
answer: string
possible_answers: list<item: string>
child 0, item: string
image: string
width: int64
height: int64
bboxs: list<item: list<item: int64>>
child 0, item: list<item: int64>
child 0, item: int64
dataset: string
split: string
to
{'question': Value(dtype='string', id=None), 'answer': Value(dtype='string', id=None), 'full_answer': Value(dtype='string', id=None), 'image': Value(dtype='string', id=None), 'width': Value(dtype='int64', id=None), 'height': Value(dtype='int64', id=None), 'bboxs': Sequence(feature=Sequence(feature=Value(dtype='int64', id=None), length=-1, id=None), length=-1, id=None), 'dataset': Value(dtype='string', id=None), 'split': Value(dtype='string', id=None), 'reasoning': [{'operation': Value(dtype='string', id=None), 'dependencies': Sequence(feature=Value(dtype='int64', id=None), length=-1, id=None), 'argument': Value(dtype='string', id=None)}], 'thought': Value(dtype='string', id=None)}
because column names don't match
During handling of the above exception, another exception occurred:
DatasetGenerationCastError Traceback (most recent call last)
Cell In[1], line 25
11 from datasets import load_dataset, Features, Value, Sequence
13 features = Features({
14 "question": Value("string"),
15 "answer": Value("string"),
(...) 22 "split": Value("string")
23 })
---> 25 dataset = load_dataset(
26 "deepcs233/Visual-CoT",
27 # features=features,
28 # trust_remote_code=True
29 )
File ~/fm-efs/znliu/miniconda3/lib/python3.13/site-packages/datasets/load.py:2084, in load_dataset(path, name, data_dir, data_files, split, cache_dir, features, download_config, download_mode, verification_mode, keep_in_memory, save_infos, revision, token, streaming, num_proc, storage_options, trust_remote_code, **config_kwargs)
2081 return builder_instance.as_streaming_dataset(split=split)
2083 # Download and prepare data
-> 2084 builder_instance.download_and_prepare(
2085 download_config=download_config,
2086 download_mode=download_mode,
2087 verification_mode=verification_mode,
2088 num_proc=num_proc,
2089 storage_options=storage_options,
2090 )
2092 # Build dataset for splits
2093 keep_in_memory = (
2094 keep_in_memory if keep_in_memory is not None else is_small_dataset(builder_instance.info.dataset_size)
2095 )
File ~/fm-efs/znliu/miniconda3/lib/python3.13/site-packages/datasets/builder.py:925, in DatasetBuilder.download_and_prepare(self, output_dir, download_config, download_mode, verification_mode, dl_manager, base_path, file_format, max_shard_size, num_proc, storage_options, **download_and_prepare_kwargs)
923 if num_proc is not None:
924 prepare_split_kwargs["num_proc"] = num_proc
--> 925 self._download_and_prepare(
926 dl_manager=dl_manager,
927 verification_mode=verification_mode,
928 **prepare_split_kwargs,
929 **download_and_prepare_kwargs,
930 )
931 # Sync info
932 self.info.dataset_size = sum(split.num_bytes for split in self.info.splits.values())
File ~/fm-efs/znliu/miniconda3/lib/python3.13/site-packages/datasets/builder.py:1001, in DatasetBuilder._download_and_prepare(self, dl_manager, verification_mode, **prepare_split_kwargs)
997 split_dict.add(split_generator.split_info)
999 try:
1000 # Prepare split will record examples associated to the split
-> 1001 self._prepare_split(split_generator, **prepare_split_kwargs)
1002 except OSError as e:
1003 raise OSError(
1004 "Cannot find data file. "
1005 + (self.manual_download_instructions or "")
1006 + "\nOriginal error:\n"
1007 + str(e)
1008 ) from None
File ~/fm-efs/znliu/miniconda3/lib/python3.13/site-packages/datasets/builder.py:1742, in ArrowBasedBuilder._prepare_split(self, split_generator, file_format, num_proc, max_shard_size)
1740 job_id = 0
1741 with pbar:
-> 1742 for job_id, done, content in self._prepare_split_single(
1743 gen_kwargs=gen_kwargs, job_id=job_id, **_prepare_split_args
1744 ):
1745 if done:
1746 result = content
File ~/fm-efs/znliu/miniconda3/lib/python3.13/site-packages/datasets/builder.py:1873, in ArrowBasedBuilder._prepare_split_single(self, gen_kwargs, fpath, file_format, max_shard_size, job_id)
1871 writer.write_table(table)
1872 except CastError as cast_error:
-> 1873 raise DatasetGenerationCastError.from_cast_error(
1874 cast_error=cast_error,
1875 builder_name=self.info.builder_name,
1876 gen_kwargs=gen_kwargs,
1877 token=self.token,
1878 )
1879 num_examples_progress_update += len(table)
1880 if time.time() > _time + config.PBAR_REFRESH_TIME_INTERVAL:
DatasetGenerationCastError: An error occurred while generating the dataset
All the data files must have the same columns, but at some point there are 1 new columns ({'possible_answers'}) and 3 missing columns ({'full_answer', 'thought', 'reasoning'}).
This happened while the json dataset builder was generating data using
hf://datasets/deepcs233/Visual-CoT/metadata/docvqa_cot_train.jsonl (at revision 223d2d8c1146fda2bb918801b8276c587b78b61c)
Please either edit the data files to have matching columns, or separate them into different configurations (see docs at https://hf.co/docs/hub/datasets-manual-configuration#multiple-configurations)
Hi,
Thanks for the great work. I encountered the following error when trying loading the dataset from HF, you can reproduce by simply running
Seems that this error is caused by the inconsistent column names in "metadata/docvqa_cot_train.jsonl", should be an easy fix.