WenJunjie Blog

sysadmin

Shell Tips

| Comments

shell tips

1. 排除重复

sort+uniq
awk '!a[$0]++'

2. find

find . type f -perm 777 -exec chmod 755 {} \;
find . type f -size 0 -exec rm -rf {} \;
find . \( -name a.out -o -name '*.o' -o -name 'core' \) -exec rm {} \;

3. 打出占用空间最大的文件或目录

du -cks * | sort -rn | head -n 10
du -shc * | sort -rn | head

du -c -h *.php
du *.php |awk 'BEGIN{count=0;size=0;} \
 {count=count+1;size=size+$1/1024;} \
 END{print "Total count " count; \
 print "Total size " size/1024 " GB"; \
 print "Avg size " size/count " MB"}'

find . -size +100k -exec ls -l {} \; | awk 'BEGIN{count=0;size=0;} \
 {count=count+1;size=size + $5/1024/1024;} \
 END{print "Total count " count; \
 print "Total size " size / 1024 " GB"; \
 print "Avg size " size / count " MB"; \
 print "--"}'

find -maxdepth 1 -type f -name "*.log.php" -mtime +21 -print \
 | xargs ls -l | awk 'BEGIN{count=0;size=0;} \
 {count=count+1;size=size + $5/1024/1024;} \
 END{print "Total count " count; \
 print "Total size " size / 1024 " GB"; \
 print "Avg size " size / count " MB"; \
 print "--"}'

4. 取IP地址

ifconfig  | grep 'inet addr:'| grep -v '127.0.0.1' | cut -d: -f2 | awk '{ print $1}'
ifconfig eth0 | sed -n '/inet /{s/.*addr://;s/ .*//;p}'
ifconfig eth0 | grep -oP '(?<=addr:).*(?=  Bcast)'

5. 过滤出登录IP

grep -oP '(\d+\.){3}\d+' /var/log/secure

6. 查看异常日志

egrep -i 'error|warn' /var/log/messages
dmesg   

7. 查看0-8点的pam认证日志

awk '$3 > "00:00" && $3 < "08:00" {print $0}' /var/log/messages |grep pam_user
sed -n '/00:00/,/08:00/ p' /var/log/messages |grep pam_user

8. 过滤日志 时间范围

grep -oP '26\/Nov\/2012(:\d{2}){3}' access.log | uniq -c | sort -n > tmp.log
grep -P '26\/Nov\/2012:15:0[0-9]:\d{2}' access.log > high.log
grep -oP '((?<=GET\s)|(?<=POST\s))[^?\s]+' high.log | sort | uniq -c | sort -n > request.log
grep -P '\/fitter\/test?' high.log \
 | grep -oP '(?<=\d\s")[^?"]+' | sort | uniq -c | sort -n > t.log
awk '/10:03:[0-9][0-9]/,/20:03:[0-9][0-9]/{print}' access.log
sed -n "/10:03:[0-9][0-9]/,/20:03:[0-9][0-9]/p" access.lg
cat file | grep -v ^# |grep -v ^$

awk '$9==500 {print $0}' access.log
awk '/2013:14:05/,0' access.log

awk -v ref="$REF" 'match($0, ref) {print $2}'

REF=SEARCH_TEXT
echo "some text" | awk "/$REF/"'{print $2}'

awk -v FTB=“$FilterTimeB” -v FTE=“$FilterTimeE” ‘$0~FTB,$0~FTE {print $0 }’ test.log | grep FINISH_LOG |awk ‘match($0,/timedelay=([0-9]+).*/,a){print a[1]}’ |awk ‘{if ($0>10) {print $0 } else {} }’

egrep “2[0-3]:[00-59]” /var/log/messages grep -E “2[0-3]:[00-59]” /var/log/messages

sed -n ’/20:00:00/,/23:59:59/p’ /var/log/messages

awk ‘$3>”20:00:00″&&$3<”24:00:00″{print }’ /var/log/messages

awk ‘{if ($4>“[25/Sep/2013:00:02:08” && $4<“[25/Sep/2013:00:03:00”) print $0 }’ appmail_access.log

awk -F’,‘ ’{if($1 ~ /^IF1012|M000300/ && $7 > ”2010/11/17 08:45:00.000” && $7 < ”2010/11/17 15:15:30.000”) print $0}‘ /home/data/vwap.tbt.20101117 awk -F’,‘ ’{if($1 ~ /^IF1012|M000300/ && $7 > “2010/11/17 09:24:30.527”) print $0}‘ /home/data/vwap.tbt.20101117 > /home/data/temp.txt

9. for

cd /usr/local/mysql/bin;for i in *;do ln /usr/local/mysql/bin/$i /usr/bin/$i;done
for i in `ls ./`; do stat ${i}/${i}_`date +%Y%m%d%H`;done
for tar in *.tar.gz; do tar zxvf $tar; done
ls *.tar.gz | xargs -n1 tar zxvf
find -maxdepth 1 -name "*.tar.gz" | xargs -i tar zxvf {}
for i in $(ls *.tar.gz); do tar zxvf $i; done

10. 取后2位数值

a=1234567&&echo ${a: 5}

11. tcp状态统计

netstat -n | awk '/^tcp/ {++S[$NF]} END {for(a in S) print a, S[a]}'
netstat -tn | awk 'NR>2{a[$NF]++}END{for(i in a)print i,a[$NF]}'
netstat -s | grep 'established'

12. 效率差

netstat -tn |tail -n+3 |awk '{print $NF}' |sort |uniq -c
netstat -tn |awk 'NR>2{print $NF}' |sort |uniq -c

13. 统计目录下文件大小,按M显示

du $1 --max-depth=1 | sort -n | awk '{printf "%7.2fM ---> %s\n",$1/1024,$2}'|sed 's:/.*/\([^/]\{1,\}\)$:\1:g'

14. 统计jpg文件大小

find . -name "*.tar.gz" -exec wc -c {} \; |awk '{print $1}' |awk '{a+=$1}END{print a/1024/1024 "MB"}'

find . -type f -size +1024k -exec ls -lh {} \; | awk '{print $9 ":"$5}'  #大于1M
du -k | sort -n | perl -ne ‘if ( /^(\d+)\s+(.*$)/){$l=log($1+.1);$m=int($l/log(1024)); printf  (“%6.1f\t%s\t%25s  %s\n”,($1/(2**(10*$m))),((“K”,”M”,”G”,”T”,”P”)[$m]),”*”x (1.5*$l),$2);}’
ls -lS | less
ls -lSr

find . -xdev -printf ‘%s %p\n’ |sort -nr|head -20
du -xak .|sort -n|tail -50

15. 查看文件大小

stat --printf="%s\n" filename

16. awk内容排重

awk '!a[$0]++' file

17. 字符串截取

testfile='/home/jack/test/file'; echo ${testfile##*/}
file
testfile='/home/jack/test/file'; echo ${file%/*}
/home/jack/test/

18. 打印奇数行

awk 'i=!i' file
awk 'NR%2' file
打印偶数行
awk '!(i=!i)' file
awk '!(NR%2)' file

19. awk所匹配行后面某一行的操作

seq 10 | awk '/4/{f=4}--f==0'
seq 10 | awk '/4/{f=4};--f==0{print}'
seq 20000 | awk '/^4$/{f=4}--f==0{print;exit}'

20. 打印匹配行和其后面的100行

grep -A100 string
sed -n '/string/,+100p'
awk '/string/{f=100}--f>=0'

21. 去除空行(空格、tab)

sed '/^\s*$/d' file
awk 'NF' file

22. 获取当前的shell

echo $0
echo $SHELL
ps -p $$ -o command= | awk '{print $1}'

23. 打印匹配字符串之间的所有行

包含匹配行
seq 10 | sed -n '/3/,/6/p'
seq 10 | awk '/3/,/6/'
不包含匹配行
seq 10 | sed -n '/3/{:f;n;/6/q;p;bf}'
seq 10 | awk '/3/{f=1;next};/6/{exit};f'

24. 临时开启http服务

python -m SimpleHTTPServer 8080
php -S localhost:80 (php-5.4)

25. lftp上传下载

lftp 192.168.1.23:21 -u user,password -e "mput file.zip;quit"
lftp 192.168.1.23:21 -u user,password -e "get file.zip;quit"

26. 批量修改文件扩展名

rename .jpeg .jpg `find . -name "*.jpeg"`
find . -type f | xargs -t -i mv {} {}_postfix
find . -type f | xargs -t rename _postfix ""
find . -type f | xargs -t rename ./ prefix_
find . -type f | xargs -t rename prefix_ ""
find . -type f | xargs -t rename run "test"
rename test run *.log

27. 压缩,去掉不需要目录

tar zcvf test.bak.tar.gz test --exclude=test/logs/*

28. 内存排序

ps -eo "%C : %p : %z : %a" | sort -k5 -rn |head
ps -e -o "%C : %p : %z : %a"|sort -rn

29. 删除当前目录test.txt以外的文件,ksh

ksh
rm -f !(test.txt)
exit

30. 杀进程

cat kill.sh
#!/bin/bash
ps -ef | grep $1 |grep -v grep | awk '{print $2}' | xargs kill -9
sh -x kill.sh php

31. 快速格式化磁盘

echo -e "o\nn\np\n1\n\n\nw\n" | fdisk /dev/sdX

32. 快速备份

cp file{,.bak}

33. 复制文件 禁用alias 直接覆盖 不提示确定

\cp file/* file

34. 删除当前目录的svn版本信息.svn

find . -name "*.svn" -type d -exec rm -rf {} \;

35. 赋予可写权限

find . -name "Cache" |xargs -t chmod -R 777

36. 将竖行变成橫行显示

sed -n 'H;${g;s/\n/ /g;p;}' file
cat file |xargs
将横行变成竖行显示
sed -n 's/ / \n/g'p file

37. 打印除第一列以外的列

awk '{$1="";print}'

38. 列求和

awk 'BEGIN{sum=0}{sum+=$1}END{print sum}'

39. cpan安装模块

cpan ExtUtils::MakeMaker File::Slurp Nagios::Plugin Nagios::Plugin::Getopt Nagios::Plugin::Threshold

40. 修改密码

echo "user:password" | chpasswd
echo "password" | passwd --stdin user

41. 找出除某目录以外的文件

find . -wholename './log' -prune -o -print
find . -path './log' -prune -o -print
ls -R -I  './log'

42. 检查网站各个连接环节的时长

curl -o /dev/null -s -w %{time_namelookup}‘\n’%{time_connect}‘\n’%{time_starttransfer}‘\n’%{size_request}‘\n’%{time_total}‘\n’ www.sina.com.cn

43. 同步目录

rsync  -vzrtopg --progress --stats -e ssh /log/  root@192.168.1.23:/log/

查硬盘类型 sas scsi sata
cat /proc/scsi/scsi | grep Model
lsmod |grep mpt

44. tcpdump

HTTP traffic including request and response headers and message body

tcpdump -A -s 0 ‘tcp port 80 and (((ip[2:2] -- ((ip[0]&0xf)<<2)) - ((tcp[12]&0xf0)>>2)) != 0)'
tcpdump -X -s 0 ‘tcp port 80 and (((ip[2:2] -- ((ip[0]&0xf)<<2)) - ((tcp[12]&0xf0)>>2)) != 0)'

tcpdump host 192.168.1.23 and imcp -i eth1

45. 统计隐藏目录大小

find /root -maxdepth 1 -name ".*"  -type d -print | xargs du -sm | sort -n | awk '{print $1"M",$2}'

46. 查询php木马

find ./ -name "*.php" -type f -print0|xargs -0 egrep "(phpspy|c99sh|milw0rm|eval\(base64_decode|spider_bc)"|awk -F: '{print $1}'|sort|uniq

47. tcpdump 抓包 ,用来防止80端口被人攻击时可以分析数据

tcpdump -c 10000 -i eth0 -n dst port 80 > /root/pkts
然后检查IP的重复数 并从小到大排序 注意 "-t\ +0"  中间是两个空格
less pkts | awk {'printf $3"\n"'} | cut -d. -f 1-4 | sort | uniq -c | awk {'printf $1" "$2"\n"'} | sort -n -t\ +0
或者用如下一句:
tcpdump -i eth0 -tnn dst port 80 -c 1000 | awk -F"." '{print $1"."$2"."$3"."$4}' | sort | uniq -c | sort -nr |head -20
tcpdump -vvX -i eth1 -s 1024 host 192.168.50.88 and port 19019 -w test9.cap
tcpdump -nn -s0 -vv  -i eth0 host 218.206.176.177 and tcp port 7890 -w xxx.cap

48. 漂亮调试代码 PS4 set -x

#!/bin/bash
set -eEu
set -o pipefail
trap 'echo >&2 "$0: unknown error ($?)"' ERR
PS4='+${BASH_SOURCE}:${LINENO}: ${FUNCNAME[0]:+${FUNCNAME[0]}(): }'; set -x

49. 删除文件最后一行,效率高,适合大文件操作

truncate -s `expr $(stat --printf="%s" a) - $(tail -n1 a|wc -c)` a

50. 闰年

date -d '2012-02-29' >&/dev/null && echo '闰年' || echo '平年'

51. sed匹配的行存在,则替换为string,不存在则插入到最后一行

echo 'test' | sed -n '/test/{h;s//string/g};p;${x;s/^$/string/p}'
echo 'haha' | sed -n '/test/{h;s//string/g};p;${x;s/^$/string/p}'

52. 把某目录下文件追加到另一目录中的同位置同名的文件

find /home/gg/2009/ -type f -printf "%P\n" \
| xargs -n 1 -i{} awk '{print >>"/home/hh/2009/{}";}' "/home/gg/2009/{}"

53. 根据掩码值计算掩码位(gawk rshift函数)

echo '255.255.255.240' | \
awk -F. '{for(i=1;i<=NF;i++){if($i==0){printf("%d\n",a);next}; \
while(!($i%2)){res++;$i=rshift($i,1)};a+=(8-res);res=0;}print a}'

54. sed的r命令可以在匹配行后插入指定文件的内容,但如果要在匹配行之前插入

sed -n '1{h;n};/aaa/r b;x;$G;p' a
sed ‘/aaa/{h;s/.*/cat b/e;G}’ a

55. 测试3306端口

echo X | telnet -e X 192.168.1.26 3306 | nc -z -w1 192.168.1.26 3306

56. 快速删除大文件

/var/spool/clientqueue/ nohup.out core –delete-before 接收者在传输之前进行删除操作 替换原理

touch blank
rsync -a --delete-before --progress --stats blank nohup.out

mkdir blank
rsync --delete-before -d blank/ /var/spool/clientmqueue/

57. 时间戳转换

stat $file |awk ‘$1==“Modify:”{print $2” “$3}’ |awk -F. ‘{print $1}’

shell: date +%s -d “2012-12-18 14:00:02” # date -s @1355810693 date +%s date -d @Unix timestamp mysql: select unix_timestamp(“2012-12-18 14:00:02”); |SELECT from_unixtime(Unix timestamp);|SELECT unix_timestamp(time); SELECT unix_timestamp(now());

58. 查出当前目录下超过500MB文件

find . -printf "%k %p\n" | sort -g -k 1,1 | awk '{if($1 > 500000) print $1/1024 "MB" " " $2 }' |tail -n 40

【Linux】的【哲学思想】: 1.一切皆文件 2.Linux主要由短小,且目的单一的程序组成(所以一个命令一般只干一件事) 3.将多个短小的程序串联起来可以完成复杂的任务 4.尽量避免捕获用户接口(尽量避免跟用户进行接触,从发起–>回车,全自动,不需要用户再操作) 5.通过文件保存软件的配置信息(只需要一个简单的文本编辑器就可以操作整个系统)

59. 时间转换

date -d “$(echo “09SEP2012:23:58:46” | sed ’s/:/ /‘)“ dconv -i ’%d%b%Y:%H:%M:%S’ “09SEP2012:23:58:46” =>2012-09-09T23:58:46 apache nginx date -d “$(echo “24/Sep/2013:00:00:00” | sed ’s/:/ /‘| sed ’s/\// /g’) 10 minute” +“%d/%b/%Y:%H:%M:%S”

fix UTF-8 files with BOM

grep -rl $‘\xEF\xBB\xBF’ .

find . -type f -exec sed ‘1s/^\xEF\xBB\xBF//’ -i.bak {} \; -exec rm {}.bak \; find -type f -print0 | xargs -0 grep -l printf ‘^\xef\xbb\xbf' | sed ‘s/^/found BOM in: /’

find empty files and remove them

find /tmp -type f -empty -delete

iostat 2 2 |grep ^xvda |awk -F”[ ]+|[-]“ ‘{if($2>20){print $1”\t”$2}}’ |sort -rnk 2 iostat 2 2|grep ^dm-|awk -F”[ ]+|[-]“ ‘{if($3>20){print $2”\t”$3}}’|sort -rnk 2

#!/bin/bash #根据iostat结果ID找出相应VM #执行sh file.sh 编号 case $1 in *) id=ls -l /dev/mapper/|awk '{if ($6=="'$*'"){print $NF}}'|awk -F-- '{print $NF}';xe vdi-list|grep -A 1 $id ;; esac

split

split Short Option Long Option Option Description -a –suffix-length=N use suffixes of length N (default 2) -b –bytes=SIZE put SIZE bytes per output file -C –line-bytes=SIZE put at most SIZE bytes of lines per output file -d –numeric-suffixes use numeric suffixes instead of alphabetic -l –lines=NUMBER put NUMBER lines per output file

多线程下载 上传

axel -a -n 5 URL aria2c -c -s 5 URL ### uget

curl -u anonymous –upload-file bigfile.tar.bz2 ftp://ftpserver.com/somedir/

<–! * liunx中常用命令小集合(方便查找) –>

利用Shell处理的方法是: %s/(‘[1-2][0-9][0-9][0-9]-[0-1][1-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]’)/to_date(\1,‘yyyy-mm-dd hh24:mi:ss’)/g

有一个文本,里面含有MySQL的日志格式,如:2010-05-14 02:06:11,需要把他转化为Oracle中的日期格式,如:to_date(‘2010-05-14 02:06:11’,‘yyyy-mm-dd hh24:mi:ss’)

Comments