git-annex_test_hardlinks

cleanup_test.sh

#!/bin/bash

datalad remove *
cd .annex-cache/
git-annex uninit
cd ..
rm -rf .annex-cache/

test_hardlinks_v_5_w_7.20190819.sh

#!/bin/bash

# Using miniconda3-4.3.30
# > conda create --name datalad
# > conda activate datalad
# In that order
# > conda install -c conda-forge git-annex=7.20190819
# > conda install -c conda-forge datalad=0.11.8

conda activate datalad

git init --bare .annex-cache
cd .annex-cache/
git-annex init
git config annex.hardlink true
# To add in global config
# [remote "cache-tmp-datasets"]
#         url = path_to/.annex-cache
#         fetch = +refs/heads/empty_branch:refs/remotes/cache-tmp-datasets/empty_branch
#         annex-cost = 70
#         annex-speculate-present = true
#         annex-pull = false
#         annex-push = false
git config -e --global
cd ..

git clone .annex-cache/ annex-cache
cd annex-cache
git checkout -b empty_branch
touch empty_file
git add empty_file
git commit -m "Empty commit"
git push origin empty_branch
cd ..
rm -rf annex-cache

datalad create dataset
cd dataset
git config annex.thin true
git config annex.hardlink true
mkdir scripts
echo "#!/bin/bash" >> scripts/create_files.sh
echo "dd if=/dev/zero of=file1 bs=1024 count=5120" >> scripts/create_files.sh
echo "dd if=/dev/zero of=file2 bs=1024 count=5120" >> scripts/create_files.sh
echo "echo 1 >> file2" >> scripts/create_files.sh
chmod +x scripts/create_files.sh
git -c annex.largefiles=nothing add scripts/create_files.sh
git commit -m "Add scripts/create_files.sh"

datalad run scripts/create_files.sh
ls -lhi
# file1
# 142184833 -r--r--r-- 1 user group 5.0M Nov 15 15:11 .git/annex/objects/Pz/X6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6
ls -lhi .git/annex/objects/Pz/X6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6
# file2
# 142184834 -r--r--r-- 1 user group 5.1M Nov 15 15:11 .git/annex/objects/F1/Qm/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5
ls -lhi .git/annex/objects/F1/Qm/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5
git-annex copy --to cache-tmp-datasets
cd ..

datalad install -s dataset dataset_clone
cd dataset_clone
git config annex.thin true
git config annex.hardlink true
git-annex copy --from cache-tmp-datasets
ls -lhi
# file1
# 142184833 -r--r--r-- 3 user group 5.0M Nov 15 15:11 .git/annex/objects/Pz/X6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6
ls -lhi .git/annex/objects/Pz/X6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6
# file2
# 142184834 -r--r--r-- 3 user group 5.1M Nov 15 15:11 .git/annex/objects/F1/Qm/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5
ls -lhi .git/annex/objects/F1/Qm/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5

git-annex direct
# Inodes are the same
# 142184833 -rw-r--r-- 3 user group 5.0M Nov 15 15:11 file1
# 142184834 -rw-r--r-- 3 user group 5.1M Nov 15 15:11 file2
ls -lhi

test_hardlinks_v_7_w_7.20190819.sh

#!/bin/bash

# Using miniconda3-4.3.30
# > conda create --name datalad
# > conda activate datalad
# In that order
# > conda install -c conda-forge git-annex=7.20190819
# > conda install -c conda-forge datalad=0.11.8

conda activate datalad

git init --bare .annex-cache
cd .annex-cache/
git-annex init
# upgrade to v 7
git-annex upgrade
git config annex.hardlink true
# [remote "cache-tmp-datasets"]
#         url = path_to/.annex-cache
#         fetch = +refs/heads/empty_branch:refs/remotes/cache-tmp-datasets/empty_branch
#         annex-cost = 70
#         annex-speculate-present = true
#         annex-pull = false
#         annex-push = false
git config -e --global
cd ..

git clone .annex-cache/ annex-cache
cd annex-cache
git checkout -b empty_branch
touch empty_file
git add empty_file
git commit -m "Empty commit"
git push origin empty_branch
cd ..
rm -rf annex-cache

datalad create dataset
cd dataset
# upgrade to v 7
git-annex upgrade
git config annex.thin true
git config annex.hardlink true
mkdir scripts
echo "#!/bin/bash" >> scripts/create_files.sh
echo "dd if=/dev/zero of=file1 bs=1024 count=5120" >> scripts/create_files.sh
echo "dd if=/dev/zero of=file2 bs=1024 count=5120" >> scripts/create_files.sh
echo "echo 1 >> file2" >> scripts/create_files.sh
chmod +x scripts/create_files.sh
git -c annex.largefiles=nothing add scripts/create_files.sh
git commit -m "Add scripts/create_files.sh"

datalad run scripts/create_files.sh
ls -lhi
# file1
# 142184833 -r--r--r-- 1 user group 5.0M Nov 15 15:11 .git/annex/objects/Pz/X6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6
ls -lhi .git/annex/objects/Pz/X6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6
# file2
# 142184834 -r--r--r-- 1 user group 5.1M Nov 15 15:11 .git/annex/objects/F1/Qm/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5
ls -lhi .git/annex/objects/F1/Qm/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5
git-annex copy --to cache-tmp-datasets
cd ..

datalad install -s dataset dataset_clone
cd dataset_clone
# upgrade to v 7
git-annex upgrade
git config annex.thin true
git config annex.hardlink true
git-annex copy --from cache-tmp-datasets
ls -lhi
# file1
# 142184833 -r--r--r-- 3 user group 5.0M Nov 15 15:11 .git/annex/objects/Pz/X6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6
ls -lhi .git/annex/objects/Pz/X6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6
# file2
# 142184834 -r--r--r-- 3 user group 5.1M Nov 15 15:11 .git/annex/objects/F1/Qm/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5
ls -lhi .git/annex/objects/F1/Qm/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5

git-annex unlock
# Inodes are different
# 142184845 -rw-r--r-- 1 user group 5.0M Nov 15 15:11 file1
# 142185655 -rw-r--r-- 1 user group 5.1M Nov 15 15:11 file2
ls -lhi
# file1
# 142184833 -r--r--r-- 3 user group 5.0M Nov 15 15:11 .git/annex/objects/Pz/X6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6
ls -lhi .git/annex/objects/Pz/X6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6
# file2
# 142184834 -r--r--r-- 3 user group 5.1M Nov 15 15:11 .git/annex/objects/F1/Qm/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5
ls -lhi .git/annex/objects/F1/Qm/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5
git-annex lock --force

git-annex unlock --fast
# Inodes are different and occurences count shows that the file in the store has been replaced
# 142197580 -rw-r--r-- 2 user group 5.0M Nov 15 15:11 file1
# 142197584 -rw-r--r-- 2 user group 5.1M Nov 15 15:11 file2
ls -lhi
# file1
# 142197580 -r--r--r-- 2 user group 5.0M Nov 15 15:11 .git/annex/objects/Pz/X6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6
ls -lhi .git/annex/objects/Pz/X6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6
# file2
# 142197584 -r--r--r-- 2 user group 5.1M Nov 15 15:11 .git/annex/objects/F1/Qm/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5
ls -lhi .git/annex/objects/F1/Qm/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5

test_hardlinks_v_7_w_7.20191106.sh

#!/bin/bash

# Using miniconda3-4.3.30
# > conda create --name datalad_w_7.20191106
# > conda activate datalad_w_7.20191106
# In that order
# > conda install -c conda-forge git-annex=7.20191106
# > conda install -c conda-forge datalad=0.11.8

conda activate datalad_w_7.20191106

git init --bare .annex-cache
cd .annex-cache/
git-annex init
git config annex.hardlink true
# [remote "cache-tmp-datasets"]
#         url = path_to/.annex-cache
#         fetch = +refs/heads/empty_branch:refs/remotes/cache-tmp-datasets/empty_branch
#         annex-cost = 70
#         annex-speculate-present = true
#         annex-pull = false
#         annex-push = false
git config -e --global
cd ..

git clone .annex-cache/ annex-cache
cd annex-cache
git checkout -b empty_branch
touch empty_file
git add empty_file
git commit -m "Empty commit"
git push origin empty_branch
cd ..
rm -rf annex-cache

datalad create dataset
cd dataset
git config annex.thin true
git config annex.hardlink true
mkdir scripts
echo "#!/bin/bash" >> scripts/create_files.sh
echo "dd if=/dev/zero of=file1 bs=1024 count=5120" >> scripts/create_files.sh
echo "dd if=/dev/zero of=file2 bs=1024 count=5120" >> scripts/create_files.sh
echo "echo 1 >> file2" >> scripts/create_files.sh
chmod +x scripts/create_files.sh
git -c annex.largefiles=nothing add scripts/create_files.sh
git commit -m "Add scripts/create_files.sh"

datalad run scripts/create_files.sh
ls -lhi
# file1
# 142184833 -r--r--r-- 1 user group 5.0M Nov 15 15:11 .git/annex/objects/Pz/X6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6
ls -lhi .git/annex/objects/Pz/X6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6
# file2
# 142184834 -r--r--r-- 1 user group 5.1M Nov 15 15:11 .git/annex/objects/F1/Qm/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5
ls -lhi .git/annex/objects/F1/Qm/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5
git-annex copy --to cache-tmp-datasets
cd ..

datalad install -s dataset dataset_clone
cd dataset_clone
git config annex.thin true
git config annex.hardlink true
git-annex copy --from cache-tmp-datasets
ls -lhi
# file1
# 142184833 -r--r--r-- 3 user group 5.0M Nov 15 15:11 .git/annex/objects/Pz/X6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6
ls -lhi .git/annex/objects/Pz/X6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6
# file2
# 142184834 -r--r--r-- 3 user group 5.1M Nov 15 15:11 .git/annex/objects/F1/Qm/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5
ls -lhi .git/annex/objects/F1/Qm/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5

git-annex unlock
# Inodes are different
# 142184845 -rw-r--r-- 1 user group 5.0M Nov 15 15:11 file1
# 142185655 -rw-r--r-- 1 user group 5.1M Nov 15 15:11 file2
ls -lhi
# file1
# 142184833 -r--r--r-- 3 user group 5.0M Nov 15 15:11 .git/annex/objects/Pz/X6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6
ls -lhi .git/annex/objects/Pz/X6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6
# file2
# 142184834 -r--r--r-- 3 user group 5.1M Nov 15 15:11 .git/annex/objects/F1/Qm/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5
ls -lhi .git/annex/objects/F1/Qm/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5
git-annex lock --force

git-annex unlock --fast
# Inodes are different
# 142197580 -rw-r--r-- 1 user group 5.0M Nov 15 15:11 file1
# 142197584 -rw-r--r-- 1 user group 5.1M Nov 15 15:11 file2
ls -lhi
# file1
# 142184833 -r--r--r-- 3 user group 5.0M Nov 15 15:11 .git/annex/objects/Pz/X6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6
ls -lhi .git/annex/objects/Pz/X6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6/MD5E-s5242880--5f363e0e58a95f06cbe9bbc662c5dfb6
# file2
# 142184834 -r--r--r-- 3 user group 5.1M Nov 15 15:11 .git/annex/objects/F1/Qm/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5
ls -lhi .git/annex/objects/F1/Qm/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5/MD5E-s5242882--30a5fdf65cd570e0592c57990c2c7dc5