Skip to content

Commit 8c8f6cc

Browse files
author
Atsushi Abe
authored
Introduce incremental index write down in XML format (#466)
1 parent 06367df commit 8c8f6cc

File tree

21 files changed

+1320
-112
lines changed

21 files changed

+1320
-112
lines changed

‎configure.ac‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,28 @@ AC_ARG_ENABLE([buggy_ifs],
184184
)
185185
AC_MSG_RESULT([$buggy_ifs])
186186

187+
dnl
188+
dnl Handle --enable-xml-indent (default:no)
189+
dnl
190+
AC_MSG_CHECKING([whether to enable xml indentation for index])
191+
AC_ARG_ENABLE([xml_indent],
192+
[AS_HELP_STRING([--enable-xml-indent],[Enable XML indentation for index])],
193+
[xml_indent=$enableval],
194+
[xml_indent=no]
195+
)
196+
AC_MSG_RESULT([$xml_indent])
197+
198+
dnl
199+
dnl Handle --enable-format-spec25 (default:no)
200+
dnl
201+
AC_MSG_CHECKING([whether to enable format spec 2.5 support or not])
202+
AC_ARG_ENABLE([format_spec25],
203+
[AS_HELP_STRING([--enable-format-spec25],[Support format spec 2.5])],
204+
[format_spec25=$enableval],
205+
[format_spec25=no]
206+
)
207+
AC_MSG_RESULT([$format_spec25])
208+
187209
dnl
188210
dnl Handle extra compile flags for tape driver
189211
dnl
@@ -499,6 +521,16 @@ then
499521
AM_CPPFLAGS="${AM_CPPFLAGS} -DPOSIXLINK_ONLY"
500522
fi
501523

524+
if test "x$xml_indent" = "xyes"
525+
then
526+
AM_CPPFLAGS="${AM_CPPFLAGS} -DINDENT_INDEXES"
527+
fi
528+
529+
if test "x$format_spec25" = "xyes"
530+
then
531+
AM_CPPFLAGS="${AM_CPPFLAGS} -DFORMAT_SPEC25"
532+
fi
533+
502534
dnl
503535
dnl Specify CPU specific optimizer options for CRC calculation
504536
dnl

‎contrib/ut-incindex/README.md‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# Unit Tests for incremental index
2+
3+
This is a directory for having unit tests for incremental index feature.
4+
5+
## How to run
6+
7+
### Basic operation test
8+
9+
1. `cd` to this directory
10+
2. Run the basic test with `./ut-basic.sh [mount_point]`
11+
- The test script formats a (filebackend) tape under `/tmp/ltfstape`, start ltfs and stop ltfs automatically.
12+
- If `[mount_point]` is not specified, the script uses `/tmp/mnt`
13+
- You can pass the specific `ltfs` binary directory with teh environmental value `LTFS_BIN_PATH`

‎contrib/ut-incindex/ut-basic.sh‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
#!/bin/sh
2+
3+
source ./utils.sh
4+
5+
MOUNTPOINT='/tmp/mnt'
6+
TAPE_PATH='/tmp/ltfstape'
7+
8+
if [ $# == 1 ]; then
9+
MOUNTPOINT=$1
10+
elif [ $# -gt 2 ]; then
11+
MOUNTPOINT=$1
12+
TAPE_PATH=$2
13+
fi
14+
15+
# Format LTFS
16+
FormatLTFS ${TAPE_PATH}
17+
if [ $? != 0 ]; then
18+
exit 1
19+
fi
20+
21+
# Launch LTFS
22+
LaunchLTFS ${MOUNTPOINT} ${TAPE_PATH}
23+
if [ $? != 0 ]; then
24+
exit 1
25+
fi
26+
27+
# 1. CREATE DIRS
28+
# Create a few dirs and files but all objects are handles by new dirs
29+
echo "1. CREATE DIRS"
30+
mkdir -p ${MOUNTPOINT}/dir1/dir11
31+
mkdir -p ${MOUNTPOINT}/dir1/dir12
32+
mkdir -p ${MOUNTPOINT}/dir2/dir21
33+
mkdir -p ${MOUNTPOINT}/dir2/dir22
34+
echo "AAA" > ${MOUNTPOINT}/dir1/file11
35+
echo "AAA" > ${MOUNTPOINT}/dir1/dir11/file111
36+
echo "AAA" > ${MOUNTPOINT}/dir1/dir11/file112
37+
IncrementalSync ${MOUNTPOINT} '1.CREATE_DIRS'
38+
if [ $? != 0 ]; then
39+
exit 1
40+
fi
41+
42+
# 2. CREATE FILES
43+
# Create files for checking file creation and directory traverse
44+
echo "2. CREATE FILES"
45+
echo "AAA" > ${MOUNTPOINT}/dir1/dir11/file113
46+
echo "AAA" > ${MOUNTPOINT}/dir1/dir12/file121
47+
echo "AAA" > ${MOUNTPOINT}/dir2/dir22/file221
48+
echo "AAA" > ${MOUNTPOINT}/dir2/file21
49+
IncrementalSync ${MOUNTPOINT} '2.CREATE_FILES'
50+
if [ $? != 0 ]; then
51+
exit 1
52+
fi
53+
54+
# 3. MODIFY FILES
55+
# Modify contents of files. Need to check /dir2 doesn't have meta-data on the incremental index
56+
echo "3. MODIFY FILES"
57+
echo "BBB" > ${MOUNTPOINT}/dir1/dir11/file111
58+
echo "BBB" > ${MOUNTPOINT}/dir2/dir22/file221
59+
echo "BBB" > ${MOUNTPOINT}/dir1/file11
60+
IncrementalSync ${MOUNTPOINT} '3.MODIFY_FILES'
61+
if [ $? != 0 ]; then
62+
exit 1
63+
fi
64+
65+
# 4. MODIFY DIRS
66+
# Modify directory's meta-data. Need to check both /dir1 and /dir1/dir11 has meta-data
67+
# on the incremental index
68+
echo "4. MODIFY DIRS"
69+
AddXattr ${MOUNTPOINT}/dir1 'ut-attr1' 'val1'
70+
echo "CCC" > ${MOUNTPOINT}/dir1/dir11/file111
71+
IncrementalSync ${MOUNTPOINT} '4.MODIFY_DIRS'
72+
if [ $? != 0 ]; then
73+
exit 1
74+
fi
75+
76+
# 5. DELETE FILES
77+
echo "5. DELETE FILES"
78+
rm ${MOUNTPOINT}/dir1/dir11/*
79+
IncrementalSync ${MOUNTPOINT} '5.DELETE_FILES'
80+
if [ $? != 0 ]; then
81+
exit 1
82+
fi
83+
84+
# 6. DELETE DIR
85+
echo "5. DELETE DIR"
86+
rm -rf ${MOUNTPOINT}/dir1/dir11
87+
IncrementalSync ${MOUNTPOINT} '6.DELETE_DIR'
88+
if [ $? != 0 ]; then
89+
exit 1
90+
fi
91+
92+
# Stop LTFS
93+
StopLTFS ${MOUNTPOINT} ${TAPE_PATH}
94+
if [ $? != 0 ]; then
95+
exit 1
96+
fi

‎contrib/ut-incindex/utils.sh‎

Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
#!/bin/sh
2+
3+
PLATFORM=`uname`
4+
ECHO='/bin/echo'
5+
6+
if [ "x${LTFS_BIN_PATH}" == 'x' ]; then
7+
LTFS_BIN_PATH='/usr/local/bin'
8+
fi
9+
10+
AddXattr()
11+
{
12+
if [ "x$1" == "x" ]; then
13+
"Need to a target to set xattr"
14+
return 1
15+
else
16+
TARGET=$1
17+
fi
18+
19+
if [ "x$2" == "x" ]; then
20+
"Need to a name to set xattr"
21+
return 1
22+
else
23+
NAME=$2
24+
fi
25+
26+
if [ "x$3" == "x" ]; then
27+
"Need to a value to set xattr"
28+
return 1
29+
else
30+
VAL=$2
31+
fi
32+
33+
${ECHO} -n "Setting a xattr ${NAME} to ${TARGET} ... "
34+
if [ "$PLATFORM" == "Darwin" ]; then
35+
/usr/bin/xattr -w ${NAME} ${VAL} ${TARGET}
36+
else
37+
/usr/bin/attr -s ${NAME} -V ${VAL} ${TARGET}
38+
fi
39+
40+
if [ $? == 0 ]; then
41+
${ECHO} "Done"
42+
return 0
43+
else
44+
${ECHO} "Failed"
45+
return 1
46+
fi
47+
48+
}
49+
50+
IncrementalSync()
51+
{
52+
if [ "x$1" == "x" ]; then
53+
MOUNTPOINT='/tmp/mnt'
54+
else
55+
MOUNTPOINT=$1
56+
fi
57+
58+
if [ "x$2" == "x" ]; then
59+
MSG='Incremental Sync'
60+
else
61+
MSG=$2
62+
fi
63+
64+
${ECHO} -n "Syncing LTFS (Incremental) ... "
65+
if [ "$PLATFORM" == "Darwin" ]; then
66+
/usr/bin/xattr -w ltfs.vendor.IBM.IncrementalSync ${MSG} ${MOUNTPOINT}
67+
else
68+
/usr/bin/attr -s ltfs.vendor.IBM.IncrementalSync -V ${MSG} ${MOUNTPOINT}
69+
fi
70+
71+
if [ $? == 0 ]; then
72+
${ECHO} "Done"
73+
return 0
74+
else
75+
${ECHO} "Failed"
76+
return 1
77+
fi
78+
}
79+
80+
FullSync()
81+
{
82+
if [ "x$1" == "x" ]; then
83+
MOUNTPOINT='/tmp/mnt'
84+
else
85+
MOUNTPOINT=$1
86+
fi
87+
88+
if [ "x$2" == "x" ]; then
89+
MSG='Full Sync'
90+
else
91+
MSG=$2
92+
fi
93+
94+
${ECHO} "Syncing LTFS (Full) ... "
95+
if [ "$PLATFORM" == "Darwin" ]; then
96+
/usr/bin/xattr -w ltfs.vendor.IBM.FullSync ${MSG} ${MOUNTPOINT}
97+
else
98+
/usr/bin/attr -s ltfs.vendor.IBM.FullSync -V ${MSG} ${MOUNTPOINT}
99+
fi
100+
101+
if [ $? == 0 ]; then
102+
${ECHO} "Done"
103+
return 0
104+
else
105+
${ECHO} "Failed"
106+
return 1
107+
fi
108+
}
109+
110+
FormatLTFS()
111+
{
112+
if [ "x$1" == "x" ]; then
113+
TAPE_PATH='/tmp/ltfstape'
114+
else
115+
TAPE_PATH=$1
116+
fi
117+
118+
if [ ! -d ${TAPE_PATH} ]; then
119+
${ECHO} "Creating tape directory for file backend: ${TAPE_PATH}"
120+
mkdir -p ${TAPE_PATH}
121+
if [ $? != 0 ]; then
122+
${ECHO} "Failed to create a tape path: ${TAPE_PATH}"
123+
return 1
124+
fi
125+
fi
126+
127+
${ECHO} "Formatting tape directory with the file backend on ${TAPE_PATH} ... "
128+
${LTFS_BIN_PATH}/mkltfs -f -e file -d ${TAPE_PATH}
129+
if [ $? != 0 ]; then
130+
${ECHO} "Failed to format a tape path: ${TAPE_PATH}"
131+
return 1
132+
fi
133+
134+
${ECHO} "Formatted the file backend on ${TAPE_PATH}"
135+
return 0
136+
}
137+
138+
LaunchLTFS()
139+
{
140+
if [ "x$1" == "x" ]; then
141+
MOUNTPOINT='/tmp/mnt'
142+
else
143+
MOUNTPOINT=$1
144+
fi
145+
146+
if [ "x$2" == "x" ]; then
147+
TAPE_PATH='/tmp/ltfstape'
148+
else
149+
TAPE_PATH=$2
150+
fi
151+
152+
if [ ! -d ${MOUNTPOINT} ]; then
153+
${ECHO} "Creating mount point for LTFS: ${MOUNTPOINT}"
154+
mkdir -p ${MOUNTPOINT}
155+
if [ $? != 0 ]; then
156+
${ECHO} "Failed to create a mount point"
157+
return 1
158+
fi
159+
fi
160+
161+
if [ ! -d ${TAPE_PATH} ]; then
162+
${ECHO} "Creating tape directory for file backend: ${TAPE_PATH}"
163+
mkdir -p ${TAPE_PATH}
164+
if [ $? != 0 ]; then
165+
${ECHO} "Failed to create a tape path: ${TAPE_PATH}"
166+
return 1
167+
fi
168+
169+
${ECHO} "Formatting tape directory with the file backend"
170+
${LTFS_BIN_PATH}/mkltfs -f -e file -d ${TAPE_PATH}
171+
if [ $? != 0 ]; then
172+
${ECHO} "Failed to format a tape path: ${TAPE_PATH}"
173+
return 1
174+
fi
175+
fi
176+
177+
${ECHO} "Launching LTFS with the file backend"
178+
${LTFS_BIN_PATH}/ltfs -o tape_backend=file -o sync_type=unmount -o devname=${TAPE_PATH} ${MOUNTPOINT}
179+
if [ $? != 0 ]; then
180+
${ECHO} "Failed to launch LTFS on ${MOUNTPOINT}"
181+
return 1
182+
fi
183+
184+
${ECHO} "LTFS is launched on ${MOUNTPOINT}"
185+
return 0
186+
}
187+
188+
StopLTFS()
189+
{
190+
if [ "x$1" == "x" ]; then
191+
MOUNTPOINT='/tmp/mnt'
192+
else
193+
MOUNTPOINT=$1
194+
fi
195+
196+
sudo umount ${MOUNTPOINT}
197+
}

‎messages/internal_error/root.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -309,6 +309,7 @@ root:table {
309309
I5048E:string{ "Unexpected partition map in a label." }
310310
I5049E:string{ "Unexpected blocksize in a label." }
311311
I5050E:string{ "Unexpected compression in a label." }
312+
I5051E:string{ "Unsupported index type is specified." }
312313

313314
// Special error codes
314315
I9997E:string{ "Child process error (ltfsck/mkltfs): %s (%d)." }

‎messages/libltfs/root.txt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -836,6 +836,14 @@ v
836836
17292I:string { "Current position is (%llu, %llu), Error position is (%llu, %llu)." }
837837
17293E:string { "UUID in the index does not match the label." }
838838

839+
17300I:string { "Wrote inc-index of %s (Gen = %lld, Part = %c, Pos = %lld, %s)." }
840+
17301I:string { "Info inc-index, Gen = %lld, Full Part = %c, Full Pos = %lld, Back Part = %c, Back Pos = %lld)." }
841+
17302E:string { "Path helper: Provided path must be an absolute path (%s)." }
842+
17303E:string { "Unexpected value was found in the reason of inc-journal entry (%d)." }
843+
17304E:string { "Unexpected value was provided to _xml_write_incremental_delete (%d)." }
844+
17305E:string { "Failed to construct a path helper (push: %d)." }
845+
17306E:string { "Failed to find a corresponded directory in path helper (push: %d)." }
846+
839847
// For Debug 19999I:string { "%s %s %d." }
840848

841849
// DO NOT place messages with IDs 20000 or higher here!

‎src/iosched/unified.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2292,7 +2292,7 @@ int _unified_write_index_after_perm(int write_ret, struct unified_data *priv)
22922292
}
22932293

22942294
ltfs_set_commit_message_reason(SYNC_WRITE_PERM, priv->vol);
2295-
ret = ltfs_write_index(ltfs_ip_id(priv->vol), SYNC_WRITE_PERM, priv->vol);
2295+
ret = ltfs_write_index(ltfs_ip_id(priv->vol), SYNC_WRITE_PERM, LTFS_FULL_INDEX, priv->vol);
22962296

22972297
return ret;
22982298
}

‎src/libltfs/arch/errormap.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -342,6 +342,7 @@ static struct error_map fuse_error_list[] = {
342342
{ LTFS_XML_WRONG_PART_MAP, "I5048E", EINVAL},
343343
{ LTFS_XML_WRONG_BLOCKSIZE, "I5049E", EINVAL},
344344
{ LTFS_XML_WRONG_COMP, "I5050E", EINVAL},
345+
{ LTFS_BAD_INDEX_TYPE, "I5051E", EINVAL},
345346
{ EDEV_NO_SENSE, "D0000E", EIO},
346347
{ EDEV_OVERRUN, "D0002E", EIO},
347348
{ EDEV_UNDERRUN, "D0003E", ENODATA},

0 commit comments

Comments
 (0)