Cannot touch files under /lib64

breun · Unread post by **breun** » Fri Jan 27, 2012 8:03 pm

I can't touch any files under /lib64 on a CentOS 5.7 x64_64 Xen guest running the latest ASL kernel. I only have this problem on one machine, though we have several others just like this one. This is also breaking yum updates for packages with files under /lib64.

Check this:

Code: Select all

# cat /etc/redhat-release 
CentOS release 5.7 (Final)
# uname -r
2.6.32.43-6.art.x86_64
# whoami
root
# stat /lib64
  File: `/lib64'
  Size: 4096      	Blocks: 8          IO Block: 4096   map
Device: ca01h/51713d	Inode: 3391489     Links: 8
Access: (0755/drwxr-xr-x)  Uid: (    0/    root)   Gid: (    0/    root)
Access: 2012-01-28 00:44:45.000000000 +0100
Modify: 2012-01-28 00:43:29.000000000 +0100
Change: 2012-01-28 00:43:29.000000000 +0100
# touch /lib64/test
touch: setting times of `/lib64/test': No such file or directory

However, I can touch a file elsewhere and then move it into /lib64:

Code: Select all

# touch /tmp/test
# mv /tmp/test /lib64/test
# stat /lib64/test
  File: `/lib64/test'
  Size: 0         	Blocks: 0          IO Block: 4096   regular empty file
Device: ca01h/51713d	Inode: 1900614     Links: 1
Access: (0644/-rw-r--r--)  Uid: (    0/    root)   Gid: (    0/    root)
Access: 2012-01-28 00:54:16.000000000 +0100
Modify: 2012-01-28 00:54:16.000000000 +0100
Change: 2012-01-28 00:54:22.000000000 +0100

What could be the reason that touching files under /lib64 (and its subdirectories) directly is not possible?

breun · Unread post by **breun** » Fri Jan 27, 2012 8:06 pm

Maybe someone can tell from this strace?

Code: Select all

# strace touch /lib64/test
execve("//bin/touch", ["touch", "/lib64/test"], [/* 22 vars */]) = 0
brk(0)                                  = 0x612510
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x6ca1f4e28000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x6ca1f4e27000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY)      = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=28554, ...}) = 0
mmap(NULL, 28554, PROT_READ, MAP_PRIVATE, 3, 0) = 0x6ca1f4e20000
close(3)                                = 0
open("/lib64/librt.so.1", O_RDONLY)     = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0 \"@\241=\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=53448, ...}) = 0
mmap(0x3da1400000, 2132936, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x6ca1f4a05000
mprotect(0x6ca1f4a0c000, 2097152, PROT_NONE) = 0
mmap(0x6ca1f4c0c000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7000) = 0x6ca1f4c0c000
close(3)                                = 0
open("/lib64/libc.so.6", O_RDONLY)      = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\220\332\201\236=\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1722304, ...}) = 0
mmap(0x3d9e800000, 3502424, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x6ca1f46ad000
mprotect(0x6ca1f47fb000, 2097152, PROT_NONE) = 0
mmap(0x6ca1f49fb000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14e000) = 0x6ca1f49fb000
mmap(0x6ca1f4a00000, 16728, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x6ca1f4a00000
close(3)                                = 0
open("/lib64/libpthread.so.0", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240W@\237=\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=145824, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x6ca1f4e1f000
mmap(0x3d9f400000, 2204528, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x6ca1f4492000
mprotect(0x6ca1f44a8000, 2093056, PROT_NONE) = 0
mmap(0x6ca1f46a7000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x15000) = 0x6ca1f46a7000
mmap(0x6ca1f46a9000, 13168, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x6ca1f46a9000
close(3)                                = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x6ca1f4e1e000
arch_prctl(ARCH_SET_FS, 0x6ca1f4e1e6e0) = 0
mprotect(0x6ca1f46a7000, 4096, PROT_READ) = 0
mprotect(0x6ca1f49fb000, 16384, PROT_READ) = 0
mprotect(0x6ca1f4c0c000, 4096, PROT_READ) = 0
mprotect(0x6ca1f4e2a000, 4096, PROT_READ) = 0
munmap(0x6ca1f4e20000, 28554)           = 0
set_tid_address(0x6ca1f4e1e770)         = 591
set_robust_list(0x6ca1f4e1e780, 0x18)   = 0
futex(0x7fffff26c92c, FUTEX_WAKE_PRIVATE, 1) = 0
rt_sigaction(SIGRTMIN, {0x6ca1f4497380, [], SA_RESTORER|SA_SIGINFO, 0x6ca1f44a0b70}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {0x6ca1f44972b0, [], SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x6ca1f44a0b70}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) = 0
brk(0)                                  = 0x612510
brk(0x633510)                           = 0x633510
brk(0x634000)                           = 0x634000
close(0)                                = 0
open("/lib64/test", O_WRONLY|O_CREAT|O_NOCTTY|O_NONBLOCK, 0666) = -1 EINVAL (Invalid argument)
futimesat(AT_FDCWD, "/lib64/test", NULL) = -1 ENOENT (No such file or directory)
write(2, "touch: ", 7touch: )                  = 7
write(2, "setting times of `/lib64/test'", 30setting times of `/lib64/test') = 30
write(2, ": No such file or directory", 27: No such file or directory) = 27
write(2, "\n", 1
)                       = 1
close(1)                                = 0
exit_group(1)                           = ?

Unread post by **mikeshinn** » Fri Jan 27, 2012 8:44 pm

Theres nothing in the kernel that can do this, so you can rule that out.

Thats a file system error. This can happen for any number of reasons, but here are few: if you have a file system mounted noatime (as touch tries to set the atime), if there is a corruption of the file system, strange permissions on the directory, extended ACLs, selinux policies, a file system that doesnt support atime, etc.

You'll want to eliminate variables and make sure your file system is setup as it came from the vendor, running clean with no errors, mounted correctly, etc.

breun · Unread post by **breun** » Sat Jan 28, 2012 4:51 am

Other directories on the same filesystem as on which /lib64 is located do work normally, but yeah, I guess I'll have to run a filesystem check to see if there's anything wrong with that particular directory.

breun · Unread post by **breun** » Sun Jan 29, 2012 7:05 am

The filesystem is not mounted noatime, there are no strange permissions (see the output of stat), SELinux is disabled...

I noticed that when I run strace touch /tmp/test (which does work) that the utimes function is called instead of the futimesat function. Any idea why that might be? http://linux.die.net/man/2/futimesat says: "This system call is obsolete. Use utimensat(2) instead."

Code: Select all

# strace touch /tmp/test
execve("//bin/touch", ["touch", "/tmp/test"], [/* 22 vars */]) = 0
brk(0)                                  = 0x613af7
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7a1ae96d6000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7a1ae96d5000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY)      = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=28554, ...}) = 0
mmap(NULL, 28554, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7a1ae96ce000
close(3)                                = 0
open("/lib64/librt.so.1", O_RDONLY)     = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0 \"@\241=\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=53448, ...}) = 0
mmap(0x3da1400000, 2132936, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7a1ae92b3000
mprotect(0x7a1ae92ba000, 2097152, PROT_NONE) = 0
mmap(0x7a1ae94ba000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7000) = 0x7a1ae94ba000
close(3)                                = 0
open("/lib64/libc.so.6", O_RDONLY)      = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\220\332\201\236=\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1722304, ...}) = 0
mmap(0x3d9e800000, 3502424, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7a1ae8f5b000
mprotect(0x7a1ae90a9000, 2097152, PROT_NONE) = 0
mmap(0x7a1ae92a9000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14e000) = 0x7a1ae92a9000
mmap(0x7a1ae92ae000, 16728, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7a1ae92ae000
close(3)                                = 0
open("/lib64/libpthread.so.0", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240W@\237=\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=145824, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7a1ae96cd000
mmap(0x3d9f400000, 2204528, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7a1ae8d40000
mprotect(0x7a1ae8d56000, 2093056, PROT_NONE) = 0
mmap(0x7a1ae8f55000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x15000) = 0x7a1ae8f55000
mmap(0x7a1ae8f57000, 13168, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7a1ae8f57000
close(3)                                = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7a1ae96cc000
arch_prctl(ARCH_SET_FS, 0x7a1ae96cc6e0) = 0
mprotect(0x7a1ae8f55000, 4096, PROT_READ) = 0
mprotect(0x7a1ae92a9000, 16384, PROT_READ) = 0
mprotect(0x7a1ae94ba000, 4096, PROT_READ) = 0
mprotect(0x7a1ae96d8000, 4096, PROT_READ) = 0
munmap(0x7a1ae96ce000, 28554)           = 0
set_tid_address(0x7a1ae96cc770)         = 9592
set_robust_list(0x7a1ae96cc780, 0x18)   = 0
futex(0x7fff5820092c, FUTEX_WAKE_PRIVATE, 1) = 0
rt_sigaction(SIGRTMIN, {0x7a1ae8d45380, [], SA_RESTORER|SA_SIGINFO, 0x7a1ae8d4eb70}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {0x7a1ae8d452b0, [], SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x7a1ae8d4eb70}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) = 0
brk(0)                                  = 0x613af7
brk(0x634af7)                           = 0x634af7
brk(0x635000)                           = 0x635000
close(0)                                = 0
open("/tmp/test", O_WRONLY|O_CREAT|O_NOCTTY|O_NONBLOCK, 0666) = 0
utimes("/proc/self/fd/0", NULL)         = 0
close(0)                                = 0
close(1)                                = 0
exit_group(0)                           = ?

/lib64 and /tmp are both on the same / filesystem.

Unread post by **mikeshinn** » Sun Jan 29, 2012 2:18 pm

I imagine thats the way they coded touch. Works for me on Centos 5.7 under Xen with the ASL kernel, maybe its a bug on your Xen host, or in your Xen hosts kernel? Or maybe a filesystem glitch? Doesnt make any sense otherwise why your filesystem would behave normally with some of the filesystem, and not other parts of the same filesystem. That sounds like either a bug in Xen, or a corruption in your filesystem. The later seems more likely, so I'd fsck the filesystem.

[root@asl-xen-test ~]# uname -a
Linux asl-xen-test.gotroot.com 2.6.32.43-6.art.x86_64 #1 SMP Thu Jul 14 14:14:48 EDT 2011 x86_64 x86_64 x86_64 GNU/Linux
[root@asl-xen-test ~]# cat /etc/redhat-release
CentOS release 5.7 (Final)
[root@asl-xen-test ~]# touch /lib64/test
[root@asl-xen-test ~]# ls -al /lib64/test
-rw-r--r-- 1 root root 0 Jan 29 13:08 /lib64/test
[root@asl-xen-test ~]# rm /lib64/test
rm: remove regular empty file `/lib64/test'? y
[root@asl-xen-test ~]# touch /lib64/test
[root@asl-xen-test ~]# ls -al /lib64/test
-rw-r--r-- 1 root root 0 Jan 29 13:08 /lib64/test

breun · Unread post by **breun** » Thu Feb 02, 2012 7:59 am

Running fsck fixed the problem.

Unread post by **mikeshinn** » Thu Feb 02, 2012 10:51 pm

Glad to hear you got it resolved. With filesystems, I always start with the easiest solution first: fsck it. If that doesnt fix it, then I know its something else (hardware, kernel, etc.), which are exceedingly rare these days (thankfully!).

breun · Unread post by **breun** » Fri Feb 03, 2012 5:04 am

fsck on a production server means downtime, so that's why I usually spend some time trying to find out if maybe it isn't a filesystem issue.

Unread post by **mikeshinn** » Fri Feb 03, 2012 7:21 pm

True, my only point being that filesystem code is pretty rock solid. If the filesystem is acting weird, and nothing else has changed (permissions, etc.) then its a corruption or hardware issue.

Atomicorp

Cannot touch files under /lib64

Cannot touch files under /lib64

Re: Cannot touch files under /lib64

Re: Cannot touch files under /lib64

Re: Cannot touch files under /lib64

Re: Cannot touch files under /lib64

Re: Cannot touch files under /lib64

Re: Cannot touch files under /lib64

Re: Cannot touch files under /lib64

Re: Cannot touch files under /lib64

Re: Cannot touch files under /lib64