(base) raphy@raohy:~$ git clone --recurse-submodules https://github.com/Coopercoppers/PFN.git
Cloning into 'PFN'...
remote: Enumerating objects: 465, done.
remote: Counting objects: 100% (195/195), done.
remote: Compressing objects: 100% (13/13), done.
remote: Total 465 (delta 183), reused 182 (delta 182), pack-reused 270 (from 1)
Receiving objects: 100% (465/465), 9.83 MiB | 31.57 MiB/s, done.
Resolving deltas: 100% (255/255), done.
(base) raphy@raohy:~$ cd PFN
(base) raphy@raohy:~/PFN$ conda create -n pfnenv python=3.7.10
Retrieving notices: done
Channels:
- defaults
Platform: linux-64
Collecting package metadata (repodata.json): done
Solving environment: done
## Package Plan ##
environment location: /home/raphy/miniconda3/envs/pfnenv
added / updated specs:
- python=3.7.10
The following packages will be downloaded:
package | build
---------------------------|-----------------
libffi-3.3 | he6710b0_2 50 KB
python-3.7.10 | h12debd9_4 44.0 MB
------------------------------------------------------------
Total: 44.0 MB
The following NEW packages will be INSTALLED:
_libgcc_mutex pkgs/main/linux-64::_libgcc_mutex-0.1-main
_openmp_mutex pkgs/main/linux-64::_openmp_mutex-5.1-1_gnu
ca-certificates pkgs/main/linux-64::ca-certificates-2024.12.31-h06a4308_0
certifi pkgs/main/linux-64::certifi-2022.12.7-py37h06a4308_0
ld_impl_linux-64 pkgs/main/linux-64::ld_impl_linux-64-2.40-h12ee557_0
libffi pkgs/main/linux-64::libffi-3.3-he6710b0_2
libgcc-ng pkgs/main/linux-64::libgcc-ng-11.2.0-h1234567_1
libgomp pkgs/main/linux-64::libgomp-11.2.0-h1234567_1
libstdcxx-ng pkgs/main/linux-64::libstdcxx-ng-11.2.0-h1234567_1
ncurses pkgs/main/linux-64::ncurses-6.4-h6a678d5_0
openssl pkgs/main/linux-64::openssl-1.1.1w-h7f8727e_0
pip pkgs/main/linux-64::pip-22.3.1-py37h06a4308_0
python pkgs/main/linux-64::python-3.7.10-h12debd9_4
readline pkgs/main/linux-64::readline-8.2-h5eee18b_0
setuptools pkgs/main/linux-64::setuptools-65.6.3-py37h06a4308_0
sqlite pkgs/main/linux-64::sqlite-3.45.3-h5eee18b_0
tk pkgs/main/linux-64::tk-8.6.14-h39e8969_0
wheel pkgs/main/linux-64::wheel-0.38.4-py37h06a4308_0
xz pkgs/main/linux-64::xz-5.6.4-h5eee18b_1
zlib pkgs/main/linux-64::zlib-1.2.13-h5eee18b_1
Proceed ([y]/n)? y
Downloading and Extracting Packages:
Preparing transaction: done
Verifying transaction: done
Executing transaction: done
#
# To activate this environment, use
#
# $ conda activate pfnenv
#
# To deactivate an active environment, use
#
# $ conda deactivate
(base) raphy@raohy:~/PFN$ conda activate pfnenv
(pfnenv) raphy@raohy:~/PFN$
(pfnenv) raphy@raohy:~/PFN$ pip install -r requirements.txt
Collecting torch==1.9.0
Downloading torch-1.9.0-cp37-cp37m-manylinux1_x86_64.whl (831.4 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 831.4/831.4 MB 2.7 MB/s eta 0:00:00
Collecting tqdm==4.51.0
Downloading tqdm-4.51.0-py2.py3-none-any.whl (70 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 71.0/71.0 kB 8.5 MB/s eta 0:00:00
Collecting numpy==1.20.2
Downloading numpy-1.20.2-cp37-cp37m-manylinux2010_x86_64.whl (15.3 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 15.3/15.3 MB 63.7 MB/s eta 0:00:00
Collecting transformers==4.9.1
Downloading transformers-4.9.1-py3-none-any.whl (2.6 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 2.6/2.6 MB 11.8 MB/s eta 0:00:00
Collecting sentencepiece==0.1.96
Downloading sentencepiece-0.1.96-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.2/1.2 MB 11.4 MB/s eta 0:00:00
Collecting typing-extensions
Downloading typing_extensions-4.7.1-py3-none-any.whl (33 kB)
Collecting filelock
Downloading filelock-3.12.2-py3-none-any.whl (10 kB)
Collecting sacremoses
Downloading sacremoses-0.0.53.tar.gz (880 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 880.6/880.6 kB 61.6 MB/s eta 0:00:00
Preparing metadata (setup.py) ... done
Collecting requests
Downloading requests-2.31.0-py3-none-any.whl (62 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 62.6/62.6 kB 12.1 MB/s eta 0:00:00
Collecting pyyaml>=5.1
Downloading PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (670 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 670.1/670.1 kB 74.2 MB/s eta 0:00:00
Collecting packaging
Downloading packaging-24.0-py3-none-any.whl (53 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 53.5/53.5 kB 5.3 MB/s eta 0:00:00
Collecting regex!=2019.12.17
Downloading regex-2024.4.16-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (761 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 761.6/761.6 kB 65.6 MB/s eta 0:00:00
Collecting importlib-metadata
Downloading importlib_metadata-6.7.0-py3-none-any.whl (22 kB)
Collecting tokenizers<0.11,>=0.10.1
Downloading tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3 MB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 3.3/3.3 MB 4.8 MB/s eta 0:00:00
Collecting huggingface-hub==0.0.12
Downloading huggingface_hub-0.0.12-py3-none-any.whl (37 kB)
Collecting zipp>=0.5
Downloading zipp-3.15.0-py3-none-any.whl (6.8 kB)
Requirement already satisfied: certifi>=2017.4.17 in /home/raphy/miniconda3/envs/pfnenv/lib/python3.7/site-packages (from requests->transformers==4.9.1->-r requirements.txt (line 4)) (2022.12.7)
Collecting charset-normalizer<4,>=2
Downloading charset_normalizer-3.4.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (138 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 138.6/138.6 kB 28.2 MB/s eta 0:00:00
Collecting idna<4,>=2.5
Downloading idna-3.10-py3-none-any.whl (70 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 70.4/70.4 kB 7.1 MB/s eta 0:00:00
Collecting urllib3<3,>=1.21.1
Downloading urllib3-2.0.7-py3-none-any.whl (124 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 124.2/124.2 kB 12.0 MB/s eta 0:00:00
Collecting six
Downloading six-1.17.0-py2.py3-none-any.whl (11 kB)
Collecting click
Downloading click-8.1.8-py3-none-any.whl (98 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 98.2/98.2 kB 18.0 MB/s eta 0:00:00
Collecting joblib
Downloading joblib-1.3.2-py3-none-any.whl (302 kB)
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 302.2/302.2 kB 52.5 MB/s eta 0:00:00
Building wheels for collected packages: sacremoses
Building wheel for sacremoses (setup.py) ... done
Created wheel for sacremoses: filename=sacremoses-0.0.53-py3-none-any.whl size=895241 sha256=c6eedcd70a460895e056293e048657522c388f8c8f3d2541934307652e00cafc
Stored in directory: /home/raphy/.cache/pip/wheels/5b/e0/77/05245143a5b31f65af6a21f7afd3219e9fa4896f918af45677
Successfully built sacremoses
Installing collected packages: tokenizers, sentencepiece, zipp, urllib3, typing-extensions, tqdm, six, regex, pyyaml, packaging, numpy, joblib, idna, filelock, charset-normalizer, torch, requests, importlib-metadata, huggingface-hub, click, sacremoses, transformers
Successfully installed charset-normalizer-3.4.1 click-8.1.8 filelock-3.12.2 huggingface-hub-0.0.12 idna-3.10 importlib-metadata-6.7.0 joblib-1.3.2 numpy-1.20.2 packaging-24.0 pyyaml-6.0.1 regex-2024.4.16 requests-2.31.0 sacremoses-0.0.53 sentencepiece-0.1.96 six-1.17.0 tokenizers-0.10.3 torch-1.9.0 tqdm-4.51.0 transformers-4.9.1 typing-extensions-4.7.1 urllib3-2.0.7 zipp-3.15.0
(pfnenv) raphy@raohy:~/PFN$
(pfnenv) raphy@raohy:~/PFN$ python main.py \
> --data WEBNLG \
> --do_train \
> --do_eval \
> --embed_mode bertcased \
> --batch_size 20 \
> --lr 0.00002 \
> --output_file webnlg_test \
> --eval_metric micro
02/16/2025 14:09:07 - INFO - __main__ - ['main.py', '--data', 'WEBNLG', '--do_train', '--do_eval', '--embed_mode', 'bertcased', '--batch_size', '20', '--lr', '0.00002', '--output_file', 'webnlg_test', '--eval_metric', 'micro']
02/16/2025 14:09:07 - INFO - __main__ - Namespace(batch_size=20, clip=0.25, data='WEBNLG', do_eval=True, do_train=True, dropconnect=0.1, dropout=0.1, embed_mode='bertcased', epoch=100, eval_batch_size=10, eval_metric='micro', hidden_size=300, linear_warmup_rate=0.0, lr=2e-05, max_seq_len=128, output_file='webnlg_test', seed=0, steps=50, weight_decay=0)
02/16/2025 14:09:07 - INFO - __main__ - ------Training------
0%| | 0/251 [00:00<?, ?it/s]
Traceback (most recent call last):
File "main.py", line 186, in <module>
for data in tqdm(train_batch):
File "/home/raphy/miniconda3/envs/pfnenv/lib/python3.7/site-packages/tqdm/std.py", line 1171, in __iter__
for obj in iterable:
File "/home/raphy/miniconda3/envs/pfnenv/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 521, in __next__
data = self._next_data()
File "/home/raphy/miniconda3/envs/pfnenv/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 561, in _next_data
data = self._dataset_fetcher.fetch(index) # may raise StopIteration
File "/home/raphy/miniconda3/envs/pfnenv/lib/python3.7/site-packages/torch/utils/data/_utils/fetch.py", line 44, in fetch
data = [self.dataset[idx] for idx in possibly_batched_index]
File "/home/raphy/miniconda3/envs/pfnenv/lib/python3.7/site-packages/torch/utils/data/_utils/fetch.py", line 44, in <listcomp>
data = [self.dataset[idx] for idx in possibly_batched_index]
File "/home/raphy/PFN/dataloader/dataloader.py", line 30, in __getitem__
bert_words = self.tokenizer.tokenize(sent_str)
AttributeError: 'dataprocess' object has no attribute 'tokenizer'
OS: Ubuntu 24.04
How to make it work?